From ba2af1dfd284c85ee30532d1647e2ff42d6ccda4 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 15 Nov 2022 17:48:13 +0100 Subject: [PATCH 01/82] wip --- .../network-dashboard/cf/NOTES.txt | 66 +++++++++++++++++ .../network-dashboard/cf/main.py | 70 +++++++++++++++++++ .../network-dashboard/cf/plugins/__init__.py | 56 +++++++++++++++ .../network-dashboard/cf/plugins/projects.py | 44 ++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/NOTES.txt create mode 100755 blueprints/cloud-operations/network-dashboard/cf/main.py create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt b/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt new file mode 100644 index 0000000000..1b8695c665 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt @@ -0,0 +1,66 @@ +# Notes + +## Inputs + +direct inputs + +- organization id +- folders (monitored) +- projects (monitored) + +derived inputs + +- projects in folders via CAI +- networks +- subnets +- routers +- peerings +- quotas +- firewall rules +- firewall policies +- routes +- routers +- dynamic routes +- peerings +- instances + +resources + +- project quota +- firewall rules in org via CAI + - key: network +- firewall policies in org via CAI + - key: network +- networks in org via CAI +- subnets in org via CAI + - key: project, network? + - computed metrics: ip usage (used, total, utilization) + - computed metrics: secondary IP ranges +- instances + - key: network + - computed metrics: instance per network (usage, limit, utilization) +- forwarding rules in org via CAI + - key: network, type + - computed metrics: fwd rule per network per type (usage, limit, utilization) +- static routes in org via CAI + - computed metrics: routes per project (usage, limit, utilization) +- dynamic routes via routers + - computed metrics: routes per project (usage, limit, utilization) + + + +## Resources and data + +- projects + - quotas + + +## Metrics + + +## Clients + +- compute +- asset inventory +- momnitoring + diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py new file mode 100755 index 0000000000..a7a4dc1d3d --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +import collections +import google.auth +import requests + +import plugins + +from google.auth.transport.requests import AuthorizedSession + +HTTP = AuthorizedSession(google.auth.default()[0]) +Q_COLLECTION = collections.deque() +RESOURCES = {} + +Resource = collections.namedtuple('Resource', 'id data') +Result = collections.namedtuple('Result', 'phase resource data') + + +def discovery_start(): + phase = plugins.Phase.DISCOVERY + for plugin in plugins.get_plugins(phase, plugins.Step.START): + for url in plugin.func(RESOURCES): + data = fetch(url) + Q_COLLECTION.append(Result(phase, plugin.resource, data)) + + +def fetch(url): + print(url) + response = HTTP.get(url) + return response.json() + + +@click.command() +@click.option('--organization', '-o', required=True, type=int, + help='GCP organization id') +@click.option('--op-project', '-op', required=True, type=str, + help='GCP monitoring project where metrics will be stored') +@click.option('--project', '-p', required=False, type=str, multiple=True, + help='GCP project id, can be specified multiple times') +@click.option('--folder', '-p', required=False, type=int, multiple=True, + help='GCP folder id, can be specified multiple times') +def main(organization=None, op_project=None, project=None, folder=None): + if organization: + RESOURCES['organization'] = Resource(organization, {}) + if folder: + RESOURCES['folders'] = [Resource(f, {}) for f in folder] + if project: + RESOURCES['project'] = [Resource(p, {}) for p in project] + + discovery_start() + + print(Q_COLLECTION) + + +if __name__ == '__main__': + main(auto_envvar_prefix='NETMON') diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py new file mode 100644 index 0000000000..70b4094add --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -0,0 +1,56 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import enum +import importlib +import itertools +import pathlib +import pkgutil + +_PLUGINS = [] + +Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') +Phase = enum.IntEnum('Phase', 'DISCOVERY COLLECTION') +Plugin = collections.namedtuple('Plugin', + 'phase step level priority resource func') +Step = enum.IntEnum('Step', 'START END') + + +class PluginError(Exception): + pass + + +def get_plugins(phase, step=None): + pred = lambda p: not (p.phase == phase and (step is None or p.step == step)) + return itertools.filterfalse(pred, _PLUGINS) + + +def register(resource, phase, step, level=Level.PRIMARY, priority=99): + + def outer(fn): + _PLUGINS.append(Plugin(phase, step, level, priority, resource, fn)) + return fn + + return outer + + +_plugins_path = str(pathlib.Path(__file__).parent) + +for mod_info in pkgutil.iter_modules([_plugins_path], 'plugins.'): + importlib.import_module(mod_info.name) + +_PLUGINS.sort() + +__all__ = ['Level', 'Phase', 'Step', 'get_plugins', 'register'] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py new file mode 100644 index 0000000000..cd107b48ea --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py @@ -0,0 +1,44 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import register, Level, Phase, Step + +LEVEL = Level.CORE +NAME = 'project' +TYPE = 'cloudresourcemanager.googleapis.com/Project' + +_CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1/folders' + '/{}/resources:searchAll' + '?assetTypes=cloudresourcemanager.googleapis.com%2FProject') + + +@register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) +def start_discovery(resources): + for f in resources.get('folders', []): + yield _CAI_URL.format(f.id) + + +@register(NAME, Phase.DISCOVERY, Step.END, LEVEL, 0) +def end_discovery(resources, data): + return + + +@register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) +def start_collection(resources): + return + + +@register(NAME, Phase.COLLECTION, Step.END, LEVEL, 0) +def end_collection(resources, metrics, data): + return From 3e3d440973c20f7b889e31826257e52c78c017a4 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 15 Nov 2022 18:08:02 +0100 Subject: [PATCH 02/82] wip --- .../network-dashboard/cf/main.py | 52 ++++++++++++++----- .../network-dashboard/cf/plugins/__init__.py | 8 ++- .../network-dashboard/cf/plugins/projects.py | 19 +++++-- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index a7a4dc1d3d..1a895ade5b 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -16,6 +16,7 @@ import click import collections import google.auth +import logging import requests import plugins @@ -26,20 +27,45 @@ Q_COLLECTION = collections.deque() RESOURCES = {} -Resource = collections.namedtuple('Resource', 'id data') Result = collections.namedtuple('Result', 'phase resource data') -def discovery_start(): - phase = plugins.Phase.DISCOVERY - for plugin in plugins.get_plugins(phase, plugins.Step.START): +def do_discovery_end(): + phase, step = plugins.Phase.DISCOVERY, plugins.Step.END + handlers = {p.resource: p.func for p in plugins.get_plugins(phase, step)} + while Q_COLLECTION: + result = Q_COLLECTION.popleft() + func = handlers.get(result.resource) + if not func: + logging.critical( + f'collection result with no handler for {result.resource}') + print(result.resource, result.data) + else: + func(RESOURCES, result.data) + + +def do_discovery_start(): + phase, step = plugins.Phase.DISCOVERY, plugins.Step.START + for plugin in plugins.get_plugins(phase, step): for url in plugin.func(RESOURCES): data = fetch(url) Q_COLLECTION.append(Result(phase, plugin.resource, data)) +def do_init(organization, folder, project): + if organization: + RESOURCES['organization'] = plugins.Resource(organization, {}) + if folder: + RESOURCES['folders'] = [plugins.Resource(f, {}) for f in folder] + if project: + RESOURCES['project'] = [plugins.Resource(p, {}) for p in project] + phase = plugins.Phase.INIT + for plugin in plugins.get_plugins(phase): + plugin.func(RESOURCES) + + def fetch(url): - print(url) + # try response = HTTP.get(url) return response.json() @@ -54,16 +80,16 @@ def fetch(url): @click.option('--folder', '-p', required=False, type=int, multiple=True, help='GCP folder id, can be specified multiple times') def main(organization=None, op_project=None, project=None, folder=None): - if organization: - RESOURCES['organization'] = Resource(organization, {}) - if folder: - RESOURCES['folders'] = [Resource(f, {}) for f in folder] - if project: - RESOURCES['project'] = [Resource(p, {}) for p in project] + logging.basicConfig(level=logging.INFO) + + do_init(organization, folder, project) + + do_discovery_start() - discovery_start() + do_discovery_end() - print(Q_COLLECTION) + import icecream + icecream.ic(RESOURCES) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 70b4094add..65bafb950a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -22,9 +22,10 @@ _PLUGINS = [] Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') -Phase = enum.IntEnum('Phase', 'DISCOVERY COLLECTION') +Phase = enum.IntEnum('Phase', 'INIT DISCOVERY COLLECTION') Plugin = collections.namedtuple('Plugin', 'phase step level priority resource func') +Resource = collections.namedtuple('Resource', 'id data') Step = enum.IntEnum('Step', 'START END') @@ -53,4 +54,7 @@ def outer(fn): _PLUGINS.sort() -__all__ = ['Level', 'Phase', 'Step', 'get_plugins', 'register'] +__all__ = [ + 'Level', 'Phase', 'PluginError', 'Resource', 'Step', 'get_plugins', + 'register' +] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py index cd107b48ea..cd905a12b4 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from . import register, Level, Phase, Step +from . import * LEVEL = Level.CORE NAME = 'project' @@ -23,15 +23,26 @@ '?assetTypes=cloudresourcemanager.googleapis.com%2FProject') +@register(NAME, Phase.INIT, Step.START) +def start_discovery(resources): + if 'projects' not in resources: + resources['projects'] = [] + + @register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) def start_discovery(resources): for f in resources.get('folders', []): yield _CAI_URL.format(f.id) -@register(NAME, Phase.DISCOVERY, Step.END, LEVEL, 0) +@register(NAME, Phase.DISCOVERY, Step.END) def end_discovery(resources, data): - return + results = data.get('results') + if not results: + raise PluginError('---') + for result in results: + project_id = result['project'].split('/')[1] + resources['projects'].append(Resource(project_id, {})) @register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) @@ -39,6 +50,6 @@ def start_collection(resources): return -@register(NAME, Phase.COLLECTION, Step.END, LEVEL, 0) +@register(NAME, Phase.COLLECTION, Step.END) def end_collection(resources, metrics, data): return From 279c529c28c6c6ce770301232f60521e3a6d4df7 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 15 Nov 2022 18:24:30 +0100 Subject: [PATCH 03/82] wip --- .../network-dashboard/cf/main.py | 6 +++--- .../network-dashboard/cf/plugins/projects.py | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 1a895ade5b..db75abc25c 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -54,11 +54,11 @@ def do_discovery_start(): def do_init(organization, folder, project): if organization: - RESOURCES['organization'] = plugins.Resource(organization, {}) + RESOURCES['organization'] = {organization: {}} if folder: - RESOURCES['folders'] = [plugins.Resource(f, {}) for f in folder] + RESOURCES['folders'] = {f: {} for f in folder} if project: - RESOURCES['project'] = [plugins.Resource(p, {}) for p in project] + RESOURCES['projects'] = {p: {} for p in project} phase = plugins.Phase.INIT for plugin in plugins.get_plugins(phase): plugin.func(RESOURCES) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py index cd905a12b4..4e2b37fb06 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging + from . import * LEVEL = Level.CORE NAME = 'project' TYPE = 'cloudresourcemanager.googleapis.com/Project' -_CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1/folders' - '/{}/resources:searchAll' - '?assetTypes=cloudresourcemanager.googleapis.com%2FProject') +CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' + '/{}/resources:searchAll' + '?assetTypes=cloudresourcemanager.googleapis.com%2FProject') @register(NAME, Phase.INIT, Step.START) @@ -31,8 +33,9 @@ def start_discovery(resources): @register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) def start_discovery(resources): - for f in resources.get('folders', []): - yield _CAI_URL.format(f.id) + for resource_type in ('projects', 'folders'): + for k in resources.get(resource_type, []): + yield CAI_URL.format(f'{resource_type}/{k}') @register(NAME, Phase.DISCOVERY, Step.END) @@ -41,8 +44,12 @@ def end_discovery(resources, data): if not results: raise PluginError('---') for result in results: - project_id = result['project'].split('/')[1] - resources['projects'].append(Resource(project_id, {})) + if result['assetType'] != TYPE: + logging.warn(f'result for wrong type {result["assetType"]}') + continue + number = result['project'].split('/')[1] + project_id = result['displayName'] + resources['projects'][project_id] = {'number': number} @register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) From 473cd99fa70cc7a712d58b2215d8a87add91d32a Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 15 Nov 2022 20:45:50 +0100 Subject: [PATCH 04/82] wip --- .../network-dashboard/cf/main.py | 37 ++++------ .../network-dashboard/cf/plugins/networks.py | 66 +++++++++++++++++ .../network-dashboard/cf/plugins/projects.py | 20 +++--- .../cf/plugins/subnetworks.py | 71 +++++++++++++++++++ .../network-dashboard/cf/plugins/utils.py | 34 +++++++++ 5 files changed, 195 insertions(+), 33 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index db75abc25c..435894d792 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -30,31 +30,24 @@ Result = collections.namedtuple('Result', 'phase resource data') -def do_discovery_end(): - phase, step = plugins.Phase.DISCOVERY, plugins.Step.END - handlers = {p.resource: p.func for p in plugins.get_plugins(phase, step)} - while Q_COLLECTION: - result = Q_COLLECTION.popleft() - func = handlers.get(result.resource) - if not func: - logging.critical( - f'collection result with no handler for {result.resource}') - print(result.resource, result.data) - else: - func(RESOURCES, result.data) - - -def do_discovery_start(): - phase, step = plugins.Phase.DISCOVERY, plugins.Step.START - for plugin in plugins.get_plugins(phase, step): - for url in plugin.func(RESOURCES): +def do_discovery(): + phase = plugins.Phase.DISCOVERY + data_handlers = { + p.resource: p.func for p in plugins.get_plugins(phase, plugins.Step.END) + } + for plugin in plugins.get_plugins(phase, plugins.Step.START): + data_handler = data_handlers.get(plugin.resource) + urls = collections.deque(plugin.func(RESOURCES)) + for url in urls: data = fetch(url) - Q_COLLECTION.append(Result(phase, plugin.resource, data)) + next_url = data_handler(RESOURCES, data, url) + if next_url: + urls.append(next_url) def do_init(organization, folder, project): if organization: - RESOURCES['organization'] = {organization: {}} + RESOURCES['organization'] = {'id': organization} if folder: RESOURCES['folders'] = {f: {} for f in folder} if project: @@ -84,9 +77,7 @@ def main(organization=None, op_project=None, project=None, folder=None): do_init(organization, folder, project) - do_discovery_start() - - do_discovery_end() + do_discovery() import icecream icecream.ic(RESOURCES) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py new file mode 100644 index 0000000000..7f118cf78d --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py @@ -0,0 +1,66 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import urllib.parse + +from . import * +from .utils import parse_cai_page_token, parse_cai_results + +LEVEL = Level.PRIMARY +NAME = 'networks' +TYPE = 'compute.googleapis.com/Network' + +CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' + '/{}/resources:searchAll' + f'?assetTypes={urllib.parse.quote(TYPE)}&pageSize=500') + + +@register(NAME, Phase.INIT, Step.START) +def start_discovery(resources): + if 'networks' not in resources: + resources['networks'] = {} + + +@register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) +def start_discovery(resources): + org_id = resources['organization']['id'] + yield CAI_URL.format(f'organizations/{org_id}') + + +@register(NAME, Phase.DISCOVERY, Step.END) +def end_discovery(resources, data, url): + for result in parse_cai_results(NAME, TYPE, data): + name = result['displayName'] + project_number = result['project'].split('/')[1] + project_id = resources['projects:number'].get(project_number) + if not project_id: + logging.info(f'skipping network {name} in {project_number}') + continue + resources['networks'][f'{project_id}/{name}'] = { + 'name': name, + 'project_id': project_id, + 'project_number': project_number + } + return parse_cai_page_token(url, data) + + +@register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) +def start_collection(resources): + return + + +@register(NAME, Phase.COLLECTION, Step.END) +def end_collection(resources, metrics, data): + return diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py index 4e2b37fb06..ddc5da519b 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py @@ -13,8 +13,10 @@ # limitations under the License. import logging +import urllib.parse from . import * +from .utils import parse_cai_page_token, parse_cai_results LEVEL = Level.CORE NAME = 'project' @@ -22,13 +24,15 @@ CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' '/{}/resources:searchAll' - '?assetTypes=cloudresourcemanager.googleapis.com%2FProject') + f'?assetTypes={urllib.parse.quote(TYPE)}&pageSize=500') @register(NAME, Phase.INIT, Step.START) def start_discovery(resources): if 'projects' not in resources: - resources['projects'] = [] + resources['projects'] = {} + if 'project_numbers' not in resources: + resources['projects:number'] = {} @register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) @@ -39,17 +43,13 @@ def start_discovery(resources): @register(NAME, Phase.DISCOVERY, Step.END) -def end_discovery(resources, data): - results = data.get('results') - if not results: - raise PluginError('---') - for result in results: - if result['assetType'] != TYPE: - logging.warn(f'result for wrong type {result["assetType"]}') - continue +def end_discovery(resources, data, url): + for result in parse_cai_results(NAME, TYPE, data): number = result['project'].split('/')[1] project_id = result['displayName'] resources['projects'][project_id] = {'number': number} + resources['projects:number'][number] = project_id + return parse_cai_page_token(url, data) @register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py new file mode 100644 index 0000000000..1adc8b83a7 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py @@ -0,0 +1,71 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import urllib.parse + +from . import * +from .utils import parse_cai_page_token, parse_cai_results + +LEVEL = Level.PRIMARY +NAME = 'subnetworks' +TYPE = 'compute.googleapis.com/Subnetwork' + +CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' + '/{}/resources:searchAll' + f'?assetTypes={urllib.parse.quote(TYPE)}&pageSize=500') + + +@register(NAME, Phase.INIT, Step.START) +def start_discovery(resources): + if 'subnetworks' not in resources: + resources['subnetworks'] = {} + + +@register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) +def start_discovery(resources): + org_id = resources['organization']['id'] + yield CAI_URL.format(f'organizations/{org_id}') + + +# {'name': '//compute.googleapis.com/projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke', 'assetType': 'compute.googleapis.com/Subnetwork', 'project': 'projects/697669426824', 'displayName': 'gke', 'description': 'Terraform-managed.', 'additionalAttributes': ['10.0.8.1'], 'location': 'europe-west1'} + + +@register(NAME, Phase.DISCOVERY, Step.END) +def end_discovery(resources, data, url): + for result in parse_cai_results(NAME, TYPE, data): + name = result['displayName'] + project_number = result['project'].split('/')[1] + project_id = resources['projects:number'].get(project_number) + if not project_id: + logging.info(f'skipping subnetwork {name} in {project_number}') + continue + resources['networks'][f'{project_id}/{name}'] = { + 'cidr_range': result['additionalAttributes'][0], + 'name': name, + 'project_id': project_id, + 'project_number': project_number, + 'region': result['location'], + } + return parse_cai_page_token(url, data) + + +@register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) +def start_collection(resources): + return + + +@register(NAME, Phase.COLLECTION, Step.END) +def end_collection(resources, metrics, data): + return diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py new file mode 100644 index 0000000000..8ca664dd46 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + + +def parse_cai_results(resource_name, resource_type, data): + results = data.get('results') + if not results: + logging.info(f'no results for {resource_name}') + return + for result in results: + if result['assetType'] != resource_type: + logging.warn(f'result for wrong type {result["assetType"]}') + continue + yield result + + +def parse_cai_page_token(url, data): + page_token = data.get('pageToken') + if page_token: + url, _, _ = url.split('&pageToken')[0] + return f'{url}&pageToken={page_token}' From 9b8c60b21644a69300d6b6b2f315457a9708ec4a Mon Sep 17 00:00:00 2001 From: Ludo Date: Wed, 16 Nov 2022 07:41:19 +0100 Subject: [PATCH 05/82] wip --- blueprints/cloud-operations/network-dashboard/cf/main.py | 8 ++++---- .../network-dashboard/cf/plugins/networks.py | 2 +- .../network-dashboard/cf/plugins/projects.py | 2 +- .../network-dashboard/cf/plugins/subnetworks.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 435894d792..c1e49cd417 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import click import collections -import google.auth import logging -import requests +import click +import google.auth import plugins from google.auth.transport.requests import AuthorizedSession @@ -38,7 +37,8 @@ def do_discovery(): for plugin in plugins.get_plugins(phase, plugins.Step.START): data_handler = data_handlers.get(plugin.resource) urls = collections.deque(plugin.func(RESOURCES)) - for url in urls: + while urls: + url = urls.popleft() data = fetch(url) next_url = data_handler(RESOURCES, data, url) if next_url: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py index 7f118cf78d..57b8d23b02 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py @@ -28,7 +28,7 @@ @register(NAME, Phase.INIT, Step.START) -def start_discovery(resources): +def init(resources): if 'networks' not in resources: resources['networks'] = {} diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py index ddc5da519b..6e995bdbd8 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py @@ -28,7 +28,7 @@ @register(NAME, Phase.INIT, Step.START) -def start_discovery(resources): +def init(resources): if 'projects' not in resources: resources['projects'] = {} if 'project_numbers' not in resources: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py index 1adc8b83a7..e1ae9b9b95 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py @@ -28,7 +28,7 @@ @register(NAME, Phase.INIT, Step.START) -def start_discovery(resources): +def init(resources): if 'subnetworks' not in resources: resources['subnetworks'] = {} From bd77b525d295114503a7c2aea4e9970819ebdccf Mon Sep 17 00:00:00 2001 From: Ludo Date: Wed, 16 Nov 2022 14:28:04 +0100 Subject: [PATCH 06/82] discovery --- .../network-dashboard/cf/main.py | 10 ++- .../network-dashboard/cf/plugins/__init__.py | 2 +- .../cf/plugins/discovery-cai-compute.py | 77 +++++++++++++++++++ ...{projects.py => discovery-cai-projects.py} | 29 ++----- .../network-dashboard/cf/plugins/networks.py | 66 ---------------- .../cf/plugins/subnetworks.py | 71 ----------------- .../network-dashboard/cf/plugins/utils.py | 14 ++-- 7 files changed, 103 insertions(+), 166 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py rename blueprints/cloud-operations/network-dashboard/cf/plugins/{projects.py => discovery-cai-projects.py} (68%) delete mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index c1e49cd417..c25f211c41 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -14,6 +14,7 @@ # limitations under the License. import collections +import json import logging import click @@ -60,7 +61,14 @@ def do_init(organization, folder, project): def fetch(url): # try response = HTTP.get(url) - return response.json() + if response.status_code != 200: + logging.critical(f'response code {response.status_code} for URL {url}') + return {} + try: + return response.json() + except json.decoder.JSONDecodeError as e: + logging.critical(f'error decoding URL {url}: {e.args[0]}') + return {} @click.command() diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 65bafb950a..b6ed1b1428 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -22,7 +22,7 @@ _PLUGINS = [] Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') -Phase = enum.IntEnum('Phase', 'INIT DISCOVERY COLLECTION') +Phase = enum.IntEnum('Phase', 'INIT DISCOVERY ENRICHMENT COLLECTION') Plugin = collections.namedtuple('Plugin', 'phase step level priority resource func') Resource = collections.namedtuple('Resource', 'id data') diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py new file mode 100644 index 0000000000..ad7249fea5 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py @@ -0,0 +1,77 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import urllib.parse + +from . import * +from .utils import parse_cai_page_token, parse_cai_results + + +CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' + '/organizations/{organization}/resources:searchAll' + '?{asset_types}&pageSize=500') +PLUGIN_NAME = 'discovery-cai-compute' +TYPES = { + 'networks': 'Network', + 'subnetworks': 'Subnetwork', + 'firewalls': 'Firewall', + 'firewall_policies': 'FirewallPolicy', + 'instances': 'Instance', + 'routes': 'Route', + 'routers': 'Router'} +NAMES = {v:k for k,v in TYPES.items()} + + +@register(PLUGIN_NAME, Phase.INIT, Step.START) +def init(resources): + for name in TYPES: + if name not in resources: + resources[name] = {} + + +@register(PLUGIN_NAME, Phase.DISCOVERY, Step.START, Level.PRIMARY, 10) +def start_discovery(resources): + organization = resources['organization']['id'] + asset_types = '&'.join( + 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) + for t in TYPES.values()) + yield CAI_URL.format(organization=organization, asset_types=asset_types) + + +@register(PLUGIN_NAME, Phase.DISCOVERY, Step.END) +def end_discovery(resources, data, url): + for result in parse_cai_results(data, PLUGIN_NAME): + name = result['displayName'] + resource_name = NAMES[result['assetType'].split('/')[1]] + resource = {'name': name} + try: + project_number = result['project'].split('/')[1] + project_id = resources['projects:number'].get(project_number) + except KeyError: + pass + else: + if not project_id: + logging.info(f'skipping resource {name} in {project_number}') + continue + resource['project_id'] = project_id + resource['project_number'] = project_number + match resource_name: + case 'firewall_policies': + resource['id'] = result['name'].split('/')[-1] + case 'subnetworks': + resource['cidr_range'] = result['additionalAttributes'][0] + resource['region'] = result['location'] + resources[resource_name][name] = resource + return parse_cai_page_token(data, url) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-projects.py similarity index 68% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-projects.py index 6e995bdbd8..e90087a915 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-projects.py @@ -12,30 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging -import urllib.parse - from . import * from .utils import parse_cai_page_token, parse_cai_results -LEVEL = Level.CORE NAME = 'project' TYPE = 'cloudresourcemanager.googleapis.com/Project' -CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' - '/{}/resources:searchAll' - f'?assetTypes={urllib.parse.quote(TYPE)}&pageSize=500') +CAI_URL = ( + 'https://content-cloudasset.googleapis.com/v1p1beta1' + '/{}/resources:searchAll' + f'?assetTypes=cloudresourcemanager.googleapis.com%2FProject&pageSize=500') @register(NAME, Phase.INIT, Step.START) def init(resources): if 'projects' not in resources: resources['projects'] = {} - if 'project_numbers' not in resources: + if 'project:numbers' not in resources: resources['projects:number'] = {} -@register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) +@register(NAME, Phase.DISCOVERY, Step.START, Level.CORE, 0) def start_discovery(resources): for resource_type in ('projects', 'folders'): for k in resources.get(resource_type, []): @@ -44,19 +41,9 @@ def start_discovery(resources): @register(NAME, Phase.DISCOVERY, Step.END) def end_discovery(resources, data, url): - for result in parse_cai_results(NAME, TYPE, data): + for result in parse_cai_results(data, NAME, TYPE): number = result['project'].split('/')[1] project_id = result['displayName'] resources['projects'][project_id] = {'number': number} resources['projects:number'][number] = project_id - return parse_cai_page_token(url, data) - - -@register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) -def start_collection(resources): - return - - -@register(NAME, Phase.COLLECTION, Step.END) -def end_collection(resources, metrics, data): - return + return parse_cai_page_token(data, url) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py deleted file mode 100644 index 57b8d23b02..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/networks.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import urllib.parse - -from . import * -from .utils import parse_cai_page_token, parse_cai_results - -LEVEL = Level.PRIMARY -NAME = 'networks' -TYPE = 'compute.googleapis.com/Network' - -CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' - '/{}/resources:searchAll' - f'?assetTypes={urllib.parse.quote(TYPE)}&pageSize=500') - - -@register(NAME, Phase.INIT, Step.START) -def init(resources): - if 'networks' not in resources: - resources['networks'] = {} - - -@register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) -def start_discovery(resources): - org_id = resources['organization']['id'] - yield CAI_URL.format(f'organizations/{org_id}') - - -@register(NAME, Phase.DISCOVERY, Step.END) -def end_discovery(resources, data, url): - for result in parse_cai_results(NAME, TYPE, data): - name = result['displayName'] - project_number = result['project'].split('/')[1] - project_id = resources['projects:number'].get(project_number) - if not project_id: - logging.info(f'skipping network {name} in {project_number}') - continue - resources['networks'][f'{project_id}/{name}'] = { - 'name': name, - 'project_id': project_id, - 'project_number': project_number - } - return parse_cai_page_token(url, data) - - -@register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) -def start_collection(resources): - return - - -@register(NAME, Phase.COLLECTION, Step.END) -def end_collection(resources, metrics, data): - return diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py deleted file mode 100644 index e1ae9b9b95..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/subnetworks.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import urllib.parse - -from . import * -from .utils import parse_cai_page_token, parse_cai_results - -LEVEL = Level.PRIMARY -NAME = 'subnetworks' -TYPE = 'compute.googleapis.com/Subnetwork' - -CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' - '/{}/resources:searchAll' - f'?assetTypes={urllib.parse.quote(TYPE)}&pageSize=500') - - -@register(NAME, Phase.INIT, Step.START) -def init(resources): - if 'subnetworks' not in resources: - resources['subnetworks'] = {} - - -@register(NAME, Phase.DISCOVERY, Step.START, LEVEL, 0) -def start_discovery(resources): - org_id = resources['organization']['id'] - yield CAI_URL.format(f'organizations/{org_id}') - - -# {'name': '//compute.googleapis.com/projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke', 'assetType': 'compute.googleapis.com/Subnetwork', 'project': 'projects/697669426824', 'displayName': 'gke', 'description': 'Terraform-managed.', 'additionalAttributes': ['10.0.8.1'], 'location': 'europe-west1'} - - -@register(NAME, Phase.DISCOVERY, Step.END) -def end_discovery(resources, data, url): - for result in parse_cai_results(NAME, TYPE, data): - name = result['displayName'] - project_number = result['project'].split('/')[1] - project_id = resources['projects:number'].get(project_number) - if not project_id: - logging.info(f'skipping subnetwork {name} in {project_number}') - continue - resources['networks'][f'{project_id}/{name}'] = { - 'cidr_range': result['additionalAttributes'][0], - 'name': name, - 'project_id': project_id, - 'project_number': project_number, - 'region': result['location'], - } - return parse_cai_page_token(url, data) - - -@register(NAME, Phase.COLLECTION, Step.START, LEVEL, 0) -def start_collection(resources): - return - - -@register(NAME, Phase.COLLECTION, Step.END) -def end_collection(resources, metrics, data): - return diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index 8ca664dd46..e931ec5c9c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -13,22 +13,24 @@ # limitations under the License. import logging +import re +RE_URL = re.compile(r'pageToken=[^&]+&?') -def parse_cai_results(resource_name, resource_type, data): + +def parse_cai_results(data, name, resource_type=None): results = data.get('results') if not results: - logging.info(f'no results for {resource_name}') + logging.info(f'no results for {name}') return for result in results: - if result['assetType'] != resource_type: + if resource_type and result['assetType'] != resource_type: logging.warn(f'result for wrong type {result["assetType"]}') continue yield result -def parse_cai_page_token(url, data): +def parse_cai_page_token(data, url): page_token = data.get('pageToken') if page_token: - url, _, _ = url.split('&pageToken')[0] - return f'{url}&pageToken={page_token}' + return RE_URL.sub(f'pageToken={page_token}&', url) From 16c26c0183ce71daad0fe7528210ef011bb7dde2 Mon Sep 17 00:00:00 2001 From: Ludo Date: Wed, 16 Nov 2022 18:41:41 +0100 Subject: [PATCH 07/82] single discovery --- .../network-dashboard/cf/main.py | 2 +- .../network-dashboard/cf/plugins/__init__.py | 2 +- .../cf/plugins/discover-cai-compute.py | 189 ++++++++++++++++++ ...i-projects.py => discover-cai-projects.py} | 7 +- .../cf/plugins/discovery-cai-compute.py | 77 ------- .../network-dashboard/cf/plugins/utils.py | 4 +- 6 files changed, 198 insertions(+), 83 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py rename blueprints/cloud-operations/network-dashboard/cf/plugins/{discovery-cai-projects.py => discover-cai-projects.py} (91%) delete mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index c25f211c41..6017f2fc7b 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -31,7 +31,7 @@ def do_discovery(): - phase = plugins.Phase.DISCOVERY + phase = plugins.Phase.DISCOVER data_handlers = { p.resource: p.func for p in plugins.get_plugins(phase, plugins.Step.END) } diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index b6ed1b1428..0726222faa 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -22,7 +22,7 @@ _PLUGINS = [] Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') -Phase = enum.IntEnum('Phase', 'INIT DISCOVERY ENRICHMENT COLLECTION') +Phase = enum.IntEnum('Phase', 'INIT DISCOVER EXTEND AGGREGATE') Plugin = collections.namedtuple('Plugin', 'phase step level priority resource func') Resource = collections.namedtuple('Resource', 'id data') diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py new file mode 100644 index 0000000000..4d33d2b2b9 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -0,0 +1,189 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import urllib.parse + +from . import * +from .utils import parse_cai_page_token, parse_cai_results + +# https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network + +CAI_URL = ('https://content-cloudasset.googleapis.com/v1' + '/organizations/{organization}/assets' + '?contentType=RESOURCE&{asset_types}&pageSize=500') +PLUGIN_NAME = 'discovery-cai-compute' +TYPES = { + 'networks': 'Network', + 'subnetworks': 'Subnetwork', + 'firewalls': 'Firewall', + 'firewall_policies': 'FirewallPolicy', + 'instances': 'Instance', + 'routes': 'Route', + 'routers': 'Router' +} +NAMES = {v: k for k, v in TYPES.items()} + + +class Skip(Exception): + pass + + +def _self_link(s): + 'Remove initial part from self links.' + return s.replace('https://www.googleapis.com/compute/v1/', '') + + +def _handle_networks(resource, data): + 'Handle network type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['peerings'] = [] + for p in data.get('peerings', []): + if p['state'] != 'ACTIVE': + continue + resource['peerings'].append({'name': p['name'], 'network': p['network']}) + resource['subnets'] = [_self_link(s) for s in data.get('subnetworks', [])] + + +def _handle_subnetworks(resource, data): + 'Handle subnetwork type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['cidr_range'] = data['ipCidrRange'] + resource['network'] = _self_link(data['network']) + resource['purpose'] = data.get('purpose') + resource['region'] = data['region'] + resource['secondary_ranges'] = [] + for s in data.get('secondaryIpRanges', []): + resource['secondary_ranges'].append({ + 'name': s['rangeName'], + 'cidr_range': s['ipCidrRange'] + }) + + +def _handle_firewalls(resource, data): + 'Handle firewall type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['network'] = _self_link(data['network']) + + +def _handle_firewall_policies(resource, data): + 'Handle firewall policy type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['num_rules'] = len(data.get('rules', [])) + resource['num_tuples'] = data.get('ruleTupleCount', 0) + + +def _handle_instances(resource, data): + 'Handle instance type resource data.' + if data['status'] != 'RUNNING': + raise Skip() + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['zone'] = data['zone'] + resource['networks'] = [] + for i in data.get('networkInterfaces', []): + resource['networks'].append({ + 'network': _self_link(i['network']), + 'subnet': _self_link(i['subnetwork']) + }) + + +def _handle_routes(resource, data): + 'Handle route type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['network'] = _self_link(data['network']) + hop = [ + a.replace('nextHop', '').lower() for a in data if a.startswith('nextHop') + ] + resource['next_hope_type'] = hop[0] + + +def _handle_routers(resource, data): + 'Handle router type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['network'] = _self_link(data['network']) + resource['region'] = data['region'].split('/')[-1] + + +def _set_parent(resource, parent, resources): + 'Extract and set resource parent.' + parent_type, parent_id = parent.split('/')[-2:] + update = None + if parent_type == 'projects': + project_id = resources['projects:number'].get(parent_id) + if project_id: + update = {'project_id': project_id, 'project_number': parent_id} + elif parent_type == 'folders': + if int(parent_id) in resources['folders']: + update = {'parent': f'{parent_type}/{parent_id}'} + elif parent_type == 'organizations': + if resources['organization']['id'] == int(parent_id): + update = {'parent': f'{parent_type}/{parent_id}'} + if update: + resource.update(update) + return update is not None + + +@register(PLUGIN_NAME, Phase.INIT, Step.START) +def init(resources): + 'Prepare the shared datastructures for asset types managed here.' + for name in TYPES: + if name not in resources: + resources[name] = {} + + +@register(PLUGIN_NAME, Phase.DISCOVER, Step.START, Level.PRIMARY, 10) +def start_discovery(resources): + 'Start discovery by returning the asset list URL for asset types.' + logging.info('discovery compute start') + organization = resources['organization']['id'] + asset_types = '&'.join( + 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) + for t in TYPES.values()) + yield CAI_URL.format(organization=organization, asset_types=asset_types) + + +@register(PLUGIN_NAME, Phase.DISCOVER, Step.END) +def end_discovery(resources, data, url): + 'Process discovery data.' + for result in parse_cai_results(data, PLUGIN_NAME, method='list'): + resource = {} + resource_data = result['resource'] + resource_name = NAMES[resource_data['discoveryName']] + parent = resource_data['parent'] + if not _set_parent(resource, parent, resources): + logging.info(f'{result["name"]} outside perimeter') + continue + extend_func = globals().get(f'_handle_{resource_name}') + if not callable(extend_func): + raise SystemExit(f'specialized function missing for {resource_name}') + try: + extend_func(resource, resource_data['data']) + except Skip: + continue + resources[resource_name][resource['self_link']] = resource + return parse_cai_page_token(data, url) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py similarity index 91% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-projects.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index e90087a915..d05f41aa48 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging + from . import * from .utils import parse_cai_page_token, parse_cai_results @@ -32,14 +34,15 @@ def init(resources): resources['projects:number'] = {} -@register(NAME, Phase.DISCOVERY, Step.START, Level.CORE, 0) +@register(NAME, Phase.DISCOVER, Step.START, Level.CORE, 0) def start_discovery(resources): + logging.info('discovery projects start') for resource_type in ('projects', 'folders'): for k in resources.get(resource_type, []): yield CAI_URL.format(f'{resource_type}/{k}') -@register(NAME, Phase.DISCOVERY, Step.END) +@register(NAME, Phase.DISCOVER, Step.END) def end_discovery(resources, data, url): for result in parse_cai_results(data, NAME, TYPE): number = result['project'].split('/')[1] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py deleted file mode 100644 index ad7249fea5..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discovery-cai-compute.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import urllib.parse - -from . import * -from .utils import parse_cai_page_token, parse_cai_results - - -CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' - '/organizations/{organization}/resources:searchAll' - '?{asset_types}&pageSize=500') -PLUGIN_NAME = 'discovery-cai-compute' -TYPES = { - 'networks': 'Network', - 'subnetworks': 'Subnetwork', - 'firewalls': 'Firewall', - 'firewall_policies': 'FirewallPolicy', - 'instances': 'Instance', - 'routes': 'Route', - 'routers': 'Router'} -NAMES = {v:k for k,v in TYPES.items()} - - -@register(PLUGIN_NAME, Phase.INIT, Step.START) -def init(resources): - for name in TYPES: - if name not in resources: - resources[name] = {} - - -@register(PLUGIN_NAME, Phase.DISCOVERY, Step.START, Level.PRIMARY, 10) -def start_discovery(resources): - organization = resources['organization']['id'] - asset_types = '&'.join( - 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) - for t in TYPES.values()) - yield CAI_URL.format(organization=organization, asset_types=asset_types) - - -@register(PLUGIN_NAME, Phase.DISCOVERY, Step.END) -def end_discovery(resources, data, url): - for result in parse_cai_results(data, PLUGIN_NAME): - name = result['displayName'] - resource_name = NAMES[result['assetType'].split('/')[1]] - resource = {'name': name} - try: - project_number = result['project'].split('/')[1] - project_id = resources['projects:number'].get(project_number) - except KeyError: - pass - else: - if not project_id: - logging.info(f'skipping resource {name} in {project_number}') - continue - resource['project_id'] = project_id - resource['project_number'] = project_number - match resource_name: - case 'firewall_policies': - resource['id'] = result['name'].split('/')[-1] - case 'subnetworks': - resource['cidr_range'] = result['additionalAttributes'][0] - resource['region'] = result['location'] - resources[resource_name][name] = resource - return parse_cai_page_token(data, url) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index e931ec5c9c..bf8a0c0c58 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -18,8 +18,8 @@ RE_URL = re.compile(r'pageToken=[^&]+&?') -def parse_cai_results(data, name, resource_type=None): - results = data.get('results') +def parse_cai_results(data, name, resource_type=None, method='search'): + results = data.get('results' if method == 'search' else 'assets') if not results: logging.info(f'no results for {name}') return From f2ae291ebc0b6736cb43534fe03b2c245cb4e99c Mon Sep 17 00:00:00 2001 From: Ludo Date: Wed, 16 Nov 2022 18:53:10 +0100 Subject: [PATCH 08/82] page token --- .../network-dashboard/cf/main.py | 5 ++-- .../cf/plugins/discover-cai-compute.py | 23 +++++++++++++------ .../network-dashboard/cf/plugins/utils.py | 6 +++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 6017f2fc7b..1292bada12 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -60,6 +60,7 @@ def do_init(organization, folder, project): def fetch(url): # try + logging.info(f'fetch {url}') response = HTTP.get(url) if response.status_code != 200: logging.critical(f'response code {response.status_code} for URL {url}') @@ -87,8 +88,8 @@ def main(organization=None, op_project=None, project=None, folder=None): do_discovery() - import icecream - icecream.ic(RESOURCES) + # import icecream + # icecream.ic(RESOURCES) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 4d33d2b2b9..0c489af2be 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -16,7 +16,7 @@ import urllib.parse from . import * -from .utils import parse_cai_page_token, parse_cai_results +from .utils import parse_cai_results # https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network @@ -148,6 +148,15 @@ def _set_parent(resource, parent, resources): return update is not None +def _url(resources): + 'Return discovery URL' + organization = resources['organization']['id'] + asset_types = '&'.join( + 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) + for t in TYPES.values()) + return CAI_URL.format(organization=organization, asset_types=asset_types) + + @register(PLUGIN_NAME, Phase.INIT, Step.START) def init(resources): 'Prepare the shared datastructures for asset types managed here.' @@ -160,11 +169,7 @@ def init(resources): def start_discovery(resources): 'Start discovery by returning the asset list URL for asset types.' logging.info('discovery compute start') - organization = resources['organization']['id'] - asset_types = '&'.join( - 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) - for t in TYPES.values()) - yield CAI_URL.format(organization=organization, asset_types=asset_types) + yield _url(resources) @register(PLUGIN_NAME, Phase.DISCOVER, Step.END) @@ -186,4 +191,8 @@ def end_discovery(resources, data, url): except Skip: continue resources[resource_name][resource['self_link']] = resource - return parse_cai_page_token(data, url) + page_token = data.get('nextPageToken') + if page_token: + logging.info('requesting next page') + url = _url(resources) + return f'{url}&pageToken={page_token}' diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index bf8a0c0c58..8002ab8d36 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -15,7 +15,7 @@ import logging import re -RE_URL = re.compile(r'pageToken=[^&]+&?') +RE_URL = re.compile(r'nextPageToken=[^&]+&?') def parse_cai_results(data, name, resource_type=None, method='search'): @@ -31,6 +31,8 @@ def parse_cai_results(data, name, resource_type=None, method='search'): def parse_cai_page_token(data, url): - page_token = data.get('pageToken') + page_token = data.get('nextPageToken') + if page_token: + logging.info(f'page token {page_token}') if page_token: return RE_URL.sub(f'pageToken={page_token}&', url) From 1a33a1b6c5b642f21330d6a685ae60752dd78e2d Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 13:25:23 +0100 Subject: [PATCH 09/82] batch requests --- .../network-dashboard/cf/MULTIPART.md | 44 +++++++++++++++ .../network-dashboard/cf/main.py | 49 +++++++++------- .../network-dashboard/cf/plugins/__init__.py | 11 ++-- .../cf/plugins/discover-cai-compute.py | 15 +++-- .../cf/plugins/discover-cai-projects.py | 17 ++++-- .../plugins/discover-compute-project-quota.py | 56 +++++++++++++++++++ .../network-dashboard/cf/plugins/utils.py | 47 ++++++++++++++++ 7 files changed, 207 insertions(+), 32 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/MULTIPART.md create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/MULTIPART.md b/blueprints/cloud-operations/network-dashboard/cf/MULTIPART.md new file mode 100644 index 0000000000..be8a227cb5 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/MULTIPART.md @@ -0,0 +1,44 @@ +==== request start ==== +uri: +method: POST +== headers start == +b'authorization': --- Token Redacted --- +b'content-length': b'466' +b'content-type': b'multipart/mixed; boundary="===============2279194272988243142=="' +b'user-agent': b'google-cloud-sdk gcloud/409.0.0 command/gcloud.compute.project-info.describe invocation-id/ecaeec2337e24223910c5efe1c6ac176 environment/None environment-version/None interactive/True from-script/False python/3.9.12 term/xterm-256color (Linux 5.10.147-20147-gbf231eecc4e8)' +== headers end == +== body start == +--===============2279194272988243142== +Content-Type: application/http +MIME-Version: 1.0 +Content-Transfer-Encoding: binary +Content-ID: <8efd7e4e-8c7a-4d3c-9d75-85e156e4231d+0> + +GET /compute/v1/projects/tf-playground-svpc-net?alt=json HTTP/1.1 +Content-Type: application/json +MIME-Version: 1.0 +content-length: 0 +user-agent: google-cloud-sdk +accept: application/json +accept-encoding: gzip, deflate +Host: compute.googleapis.com + +--===============2279194272988243142==-- + +== body end == +==== request end ==== + +>>> from email.mime.multipart import MIMEMultipart, MIMEBase +>>> msg = MIMEMultipart("mixed") +>>> base = MIMEBase("application", "html") +>>> base.set_payload('''GET /compute/v1/projects/tf-playground-svpc-net?alt=json HTTP/1.1 +... Content-Type: application/json +... MIME-Version: 1.0 +... content-length: 0 +... user-agent: google-cloud-sdk +... accept: application/json +... accept-encoding: gzip, deflate +... Host: compute.googleapis.com +... ''') +>>> msg.attach(base) +>>> msg.as_string() diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 1292bada12..d05a9ea86f 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -36,14 +36,19 @@ def do_discovery(): p.resource: p.func for p in plugins.get_plugins(phase, plugins.Step.END) } for plugin in plugins.get_plugins(phase, plugins.Step.START): + logging.info(f'discovery {plugin.resource}') data_handler = data_handlers.get(plugin.resource) - urls = collections.deque(plugin.func(RESOURCES)) - while urls: - url = urls.popleft() - data = fetch(url) - next_url = data_handler(RESOURCES, data, url) - if next_url: - urls.append(next_url) + requests = collections.deque(plugin.func(RESOURCES)) + while requests: + request = requests.popleft() + response = fetch(request) + if not data_handler: + logging.warn(f'no discovery data handler for {plugin.resource}') + for next_request in data_handler(RESOURCES, response): + if not next_request: + continue + logging.info(f'next request {next_request}') + requests.append(next_request) def do_init(organization, folder, project): @@ -58,18 +63,24 @@ def do_init(organization, folder, project): plugin.func(RESOURCES) -def fetch(url): +def fetch(request): # try - logging.info(f'fetch {url}') - response = HTTP.get(url) + logging.info(f'fetch {request.url}') + if not request.data: + response = HTTP.get(request.url, headers=request.headers) + else: + response = HTTP.post(request.url, headers=request.headers, + data=request.data) if response.status_code != 200: - logging.critical(f'response code {response.status_code} for URL {url}') - return {} - try: - return response.json() - except json.decoder.JSONDecodeError as e: - logging.critical(f'error decoding URL {url}: {e.args[0]}') - return {} + # TODO: handle this + logging.critical( + f'response code {response.status_code} for URL {request.url}') + print(request.url) + print(request.headers) + print(request.data) + print(response.headers) + print(response.content) + return response @click.command() @@ -88,8 +99,8 @@ def main(organization=None, op_project=None, project=None, folder=None): do_discovery() - # import icecream - # icecream.ic(RESOURCES) + import icecream + icecream.ic(RESOURCES) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 0726222faa..80d4de9bc7 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -19,8 +19,14 @@ import pathlib import pkgutil +__all__ = [ + 'HTTPRequest', 'Level', 'Phase', 'PluginError', 'Resource', 'Step', + 'get_plugins', 'register' +] + _PLUGINS = [] +HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data') Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') Phase = enum.IntEnum('Phase', 'INIT DISCOVER EXTEND AGGREGATE') Plugin = collections.namedtuple('Plugin', @@ -53,8 +59,3 @@ def outer(fn): importlib.import_module(mod_info.name) _PLUGINS.sort() - -__all__ = [ - 'Level', 'Phase', 'PluginError', 'Resource', 'Step', 'get_plugins', - 'register' -] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 0c489af2be..2febe80aee 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import urllib.parse -from . import * +from . import Level, Phase, HTTPRequest, Step, register from .utils import parse_cai_results # https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network @@ -169,12 +170,18 @@ def init(resources): def start_discovery(resources): 'Start discovery by returning the asset list URL for asset types.' logging.info('discovery compute start') - yield _url(resources) + yield HTTPRequest(_url(resources), {}, None) @register(PLUGIN_NAME, Phase.DISCOVER, Step.END) -def end_discovery(resources, data, url): +def end_discovery(resources, response): 'Process discovery data.' + request = response.request + try: + data = response.json() + except json.decoder.JSONDecodeError as e: + logging.critical(f'error decoding URL {request.url}: {e.args[0]}') + return {} for result in parse_cai_results(data, PLUGIN_NAME, method='list'): resource = {} resource_data = result['resource'] @@ -195,4 +202,4 @@ def end_discovery(resources, data, url): if page_token: logging.info('requesting next page') url = _url(resources) - return f'{url}&pageToken={page_token}' + yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index d05f41aa48..bd996b77e0 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging -from . import * +from . import Level, Phase, HTTPRequest, Step, register from .utils import parse_cai_page_token, parse_cai_results NAME = 'project' @@ -39,14 +40,22 @@ def start_discovery(resources): logging.info('discovery projects start') for resource_type in ('projects', 'folders'): for k in resources.get(resource_type, []): - yield CAI_URL.format(f'{resource_type}/{k}') + yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) @register(NAME, Phase.DISCOVER, Step.END) -def end_discovery(resources, data, url): +def end_discovery(resources, response): + request = response.request + try: + data = response.json() + except json.decoder.JSONDecodeError as e: + logging.critical(f'error decoding URL {request.url}: {e.args[0]}') + return {} for result in parse_cai_results(data, NAME, TYPE): number = result['project'].split('/')[1] project_id = result['displayName'] resources['projects'][project_id] = {'number': number} resources['projects:number'][number] = project_id - return parse_cai_page_token(data, url) + next_url = parse_cai_page_token(data, request.url) + if next_url: + yield HTTPRequest(next_url, {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py new file mode 100644 index 0000000000..d00d8f8f83 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py @@ -0,0 +1,56 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from . import Level, Phase, PluginError, Step, register +from .utils import dirty_mp_request, dirty_mp_response + +NAME = 'project-quota' + +API_GLOBAL_URL = '/compute/v1/projects/{}' +API_REGION_URL = '/compute/v1/projects/{}/regions/{}' + + +@register(NAME, Phase.INIT, Step.START) +def init(resources): + if 'project-quota' not in resources: + resources['project-quota'] = {} + + +@register(NAME, Phase.DISCOVER, Step.START, Level.DERIVED, 0) +def start_discovery(resources): + yield dirty_mp_request( + [API_GLOBAL_URL.format(p) for p in resources['projects']]) + + +@register(NAME, Phase.DISCOVER, Step.END) +def end_discovery(resources, response): + content_type = response.headers['content-type'] + for part in dirty_mp_response(content_type, response.content): + kind = part.get('kind') + quota = part.get('quotas') + self_link = part.get('selfLink') + if not self_link: + logging.warn('invalid quota response') + self_link = self_link.split('/') + if kind == 'compute#project': + project_id = self_link[-1] + region = 'global' + elif kind == 'compute#region': + project_id = self_link[-3] + region = self_link[-1] + project_quota = resources['project-quota'].setdefault(project_id, {}) + project_quota[region] = quota + yield \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index 8002ab8d36..b60394a2ae 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -12,9 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import re +from . import HTTPRequest, PluginError + +MP_PART = '''\ +Content-Type: application/http +MIME-Version: 1.0 +Content-Transfer-Encoding: binary + +GET {}?alt=json HTTP/1.1 +Content-Type: application/json +MIME-Version: 1.0 +Content-Length: 0 +Accept: application/json +Accept-Encoding: gzip, deflate +Host: compute.googleapis.com + +''' RE_URL = re.compile(r'nextPageToken=[^&]+&?') @@ -36,3 +53,33 @@ def parse_cai_page_token(data, url): logging.info(f'page token {page_token}') if page_token: return RE_URL.sub(f'pageToken={page_token}&', url) + + +def dirty_mp_request(urls, boundary='1234567890'): + boundary = f'--{boundary}' + data = [boundary] + for url in urls: + data += ['\n', MP_PART.format(url), boundary] + data.append('--\n') + headers = {'content-type': f'multipart/mixed; boundary={boundary[2:]}'} + return HTTPRequest('https://compute.googleapis.com/batch/compute/v1', headers, + ''.join(data)) + + +def dirty_mp_response(content_type, content): + try: + _, boundary = content_type.split('=') + except ValueError: + raise PluginError('no boundary found in content type') + content = content.decode('utf-8').strip()[:-2] + if boundary not in content: + raise PluginError('MIME boundary not found') + for part in content.split(f'--{boundary}'): + part = part.strip() + if not part: + continue + try: + mime_header, header, body = part.split('\r\n\r\n', 3) + except ValueError: + raise PluginError('cannot parse MIME part') + yield json.loads(body) From 4d5e93683e50e97092d570a2cb039cb8ef4b0341 Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 15:08:18 +0100 Subject: [PATCH 10/82] remove plugin name --- .../network-dashboard/cf/main.py | 11 +++++++---- .../network-dashboard/cf/plugins/__init__.py | 9 ++++----- .../cf/plugins/discover-cai-compute.py | 9 ++++----- .../cf/plugins/discover-cai-projects.py | 16 ++++++++-------- .../cf/plugins/discover-compute-project-quota.py | 12 ++++++------ 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index d05a9ea86f..e4d8bd869f 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -33,17 +33,20 @@ def do_discovery(): phase = plugins.Phase.DISCOVER data_handlers = { - p.resource: p.func for p in plugins.get_plugins(phase, plugins.Step.END) + p.func.__module__: p.func + for p in plugins.get_plugins(phase, plugins.Step.END) } for plugin in plugins.get_plugins(phase, plugins.Step.START): - logging.info(f'discovery {plugin.resource}') - data_handler = data_handlers.get(plugin.resource) + plugin_name = plugin.func.__module__ + logging.info(f'discovery {plugin_name}') + data_handler = data_handlers.get(plugin_name) requests = collections.deque(plugin.func(RESOURCES)) while requests: request = requests.popleft() response = fetch(request) if not data_handler: - logging.warn(f'no discovery data handler for {plugin.resource}') + logging.warn(f'no discovery data handler for {plugin_name}') + continue for next_request in data_handler(RESOURCES, response): if not next_request: continue diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 80d4de9bc7..6f4fe22a7b 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -29,8 +29,7 @@ HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data') Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') Phase = enum.IntEnum('Phase', 'INIT DISCOVER EXTEND AGGREGATE') -Plugin = collections.namedtuple('Plugin', - 'phase step level priority resource func') +Plugin = collections.namedtuple('Plugin', 'phase step level priority func') Resource = collections.namedtuple('Resource', 'id data') Step = enum.IntEnum('Step', 'START END') @@ -44,10 +43,10 @@ def get_plugins(phase, step=None): return itertools.filterfalse(pred, _PLUGINS) -def register(resource, phase, step, level=Level.PRIMARY, priority=99): +def register(phase, step, level=Level.PRIMARY, priority=99): def outer(fn): - _PLUGINS.append(Plugin(phase, step, level, priority, resource, fn)) + _PLUGINS.append(Plugin(phase, step, level, priority, fn)) return fn return outer @@ -58,4 +57,4 @@ def outer(fn): for mod_info in pkgutil.iter_modules([_plugins_path], 'plugins.'): importlib.import_module(mod_info.name) -_PLUGINS.sort() +_PLUGINS.sort(key=lambda i: i[:-1]) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 2febe80aee..2d326938f2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -24,7 +24,6 @@ CAI_URL = ('https://content-cloudasset.googleapis.com/v1' '/organizations/{organization}/assets' '?contentType=RESOURCE&{asset_types}&pageSize=500') -PLUGIN_NAME = 'discovery-cai-compute' TYPES = { 'networks': 'Network', 'subnetworks': 'Subnetwork', @@ -158,7 +157,7 @@ def _url(resources): return CAI_URL.format(organization=organization, asset_types=asset_types) -@register(PLUGIN_NAME, Phase.INIT, Step.START) +@register(Phase.INIT, Step.START) def init(resources): 'Prepare the shared datastructures for asset types managed here.' for name in TYPES: @@ -166,14 +165,14 @@ def init(resources): resources[name] = {} -@register(PLUGIN_NAME, Phase.DISCOVER, Step.START, Level.PRIMARY, 10) +@register(Phase.DISCOVER, Step.START, Level.PRIMARY, 10) def start_discovery(resources): 'Start discovery by returning the asset list URL for asset types.' logging.info('discovery compute start') yield HTTPRequest(_url(resources), {}, None) -@register(PLUGIN_NAME, Phase.DISCOVER, Step.END) +@register(Phase.DISCOVER, Step.END) def end_discovery(resources, response): 'Process discovery data.' request = response.request @@ -182,7 +181,7 @@ def end_discovery(resources, response): except json.decoder.JSONDecodeError as e: logging.critical(f'error decoding URL {request.url}: {e.args[0]}') return {} - for result in parse_cai_results(data, PLUGIN_NAME, method='list'): + for result in parse_cai_results(data, 'cai-compute', method='list'): resource = {} resource_data = result['resource'] resource_name = NAMES[resource_data['discoveryName']] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index bd996b77e0..c0e82778e5 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -18,7 +18,7 @@ from . import Level, Phase, HTTPRequest, Step, register from .utils import parse_cai_page_token, parse_cai_results -NAME = 'project' +NAME = 'projects' TYPE = 'cloudresourcemanager.googleapis.com/Project' CAI_URL = ( @@ -27,23 +27,23 @@ f'?assetTypes=cloudresourcemanager.googleapis.com%2FProject&pageSize=500') -@register(NAME, Phase.INIT, Step.START) +@register(Phase.INIT, Step.START) def init(resources): - if 'projects' not in resources: - resources['projects'] = {} + if NAME not in resources: + resources[NAME] = {} if 'project:numbers' not in resources: resources['projects:number'] = {} -@register(NAME, Phase.DISCOVER, Step.START, Level.CORE, 0) +@register(Phase.DISCOVER, Step.START, Level.CORE, 0) def start_discovery(resources): logging.info('discovery projects start') - for resource_type in ('projects', 'folders'): + for resource_type in (NAME, 'folders'): for k in resources.get(resource_type, []): yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) -@register(NAME, Phase.DISCOVER, Step.END) +@register(Phase.DISCOVER, Step.END) def end_discovery(resources, response): request = response.request try: @@ -54,7 +54,7 @@ def end_discovery(resources, response): for result in parse_cai_results(data, NAME, TYPE): number = result['project'].split('/')[1] project_id = result['displayName'] - resources['projects'][project_id] = {'number': number} + resources[NAME][project_id] = {'number': number} resources['projects:number'][number] = project_id next_url = parse_cai_page_token(data, request.url) if next_url: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py index d00d8f8f83..1522df0b92 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py @@ -23,19 +23,19 @@ API_REGION_URL = '/compute/v1/projects/{}/regions/{}' -@register(NAME, Phase.INIT, Step.START) +@register(Phase.INIT, Step.START) def init(resources): - if 'project-quota' not in resources: - resources['project-quota'] = {} + if NAME not in resources: + resources[NAME] = {} -@register(NAME, Phase.DISCOVER, Step.START, Level.DERIVED, 0) +@register(Phase.DISCOVER, Step.START, Level.DERIVED, 0) def start_discovery(resources): yield dirty_mp_request( [API_GLOBAL_URL.format(p) for p in resources['projects']]) -@register(NAME, Phase.DISCOVER, Step.END) +@register(Phase.DISCOVER, Step.END) def end_discovery(resources, response): content_type = response.headers['content-type'] for part in dirty_mp_response(content_type, response.content): @@ -51,6 +51,6 @@ def end_discovery(resources, response): elif kind == 'compute#region': project_id = self_link[-3] region = self_link[-1] - project_quota = resources['project-quota'].setdefault(project_id, {}) + project_quota = resources[NAME].setdefault(project_id, {}) project_quota[region] = quota yield \ No newline at end of file From 5f9c4064f3e6a18e39c5eefc291c574a9e4d9d47 Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 15:26:19 +0100 Subject: [PATCH 11/82] streamline --- .../network-dashboard/cf/main.py | 19 +---- .../network-dashboard/cf/plugins/__init__.py | 44 +++++++---- .../cf/plugins/discover-cai-compute.py | 75 +++++++++---------- .../cf/plugins/discover-cai-projects.py | 37 +++++---- .../plugins/discover-compute-project-quota.py | 31 ++++---- 5 files changed, 104 insertions(+), 102 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index e4d8bd869f..757ab14558 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -31,23 +31,13 @@ def do_discovery(): - phase = plugins.Phase.DISCOVER - data_handlers = { - p.func.__module__: p.func - for p in plugins.get_plugins(phase, plugins.Step.END) - } - for plugin in plugins.get_plugins(phase, plugins.Step.START): - plugin_name = plugin.func.__module__ - logging.info(f'discovery {plugin_name}') - data_handler = data_handlers.get(plugin_name) + for plugin in plugins.get_discovery_plugins(): + logging.info(f'discovery {plugin.name}') requests = collections.deque(plugin.func(RESOURCES)) while requests: request = requests.popleft() response = fetch(request) - if not data_handler: - logging.warn(f'no discovery data handler for {plugin_name}') - continue - for next_request in data_handler(RESOURCES, response): + for next_request in plugin.handler(RESOURCES, response): if not next_request: continue logging.info(f'next request {next_request}') @@ -61,8 +51,7 @@ def do_init(organization, folder, project): RESOURCES['folders'] = {f: {} for f in folder} if project: RESOURCES['projects'] = {p: {} for p in project} - phase = plugins.Phase.INIT - for plugin in plugins.get_plugins(phase): + for plugin in plugins.get_init_plugins(): plugin.func(RESOURCES) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 6f4fe22a7b..d867214fa1 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -20,34 +20,50 @@ import pkgutil __all__ = [ - 'HTTPRequest', 'Level', 'Phase', 'PluginError', 'Resource', 'Step', - 'get_plugins', 'register' + 'HTTPRequest', 'Level', 'PluginError', 'Resource', 'get_discovery_plugins', + 'get_init_plugins', 'register_discovery', 'register_init' ] -_PLUGINS = [] +_PLUGINS_INIT = [] +_PLUGINS_DISCOVERY = [] HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data') Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') -Phase = enum.IntEnum('Phase', 'INIT DISCOVER EXTEND AGGREGATE') -Plugin = collections.namedtuple('Plugin', 'phase step level priority func') +Plugin = collections.namedtuple('Plugin', 'func name level priority handler', + defaults=[None, None, None]) Resource = collections.namedtuple('Resource', 'id data') -Step = enum.IntEnum('Step', 'START END') class PluginError(Exception): pass -def get_plugins(phase, step=None): - pred = lambda p: not (p.phase == phase and (step is None or p.step == step)) - return itertools.filterfalse(pred, _PLUGINS) +def get_discovery_plugins(): + for p in _PLUGINS_DISCOVERY: + yield p -def register(phase, step, level=Level.PRIMARY, priority=99): +def get_init_plugins(): + for p in _PLUGINS_INIT: + yield p - def outer(fn): - _PLUGINS.append(Plugin(phase, step, level, priority, fn)) - return fn + +def register_discovery(handler_func, level=Level.PRIMARY, priority=99): + + def outer(func): + _PLUGINS_DISCOVERY.append( + Plugin(func, func.__module__, level, priority, handler_func)) + return func + + return outer + + +def register_init(): + # TODO: make decorator work without args + + def outer(func): + _PLUGINS_INIT.append(Plugin(func, func.__module__)) + return func return outer @@ -57,4 +73,4 @@ def outer(fn): for mod_info in pkgutil.iter_modules([_plugins_path], 'plugins.'): importlib.import_module(mod_info.name) -_PLUGINS.sort(key=lambda i: i[:-1]) +_PLUGINS_DISCOVERY.sort(key=lambda i: i[2:-1]) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 2d326938f2..01f300ad79 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -16,7 +16,7 @@ import logging import urllib.parse -from . import Level, Phase, HTTPRequest, Step, register +from . import HTTPRequest, Level, register_init, register_discovery from .utils import parse_cai_results # https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network @@ -40,9 +40,35 @@ class Skip(Exception): pass -def _self_link(s): - 'Remove initial part from self links.' - return s.replace('https://www.googleapis.com/compute/v1/', '') +def _handle_discovery(resources, response): + 'Process discovery data.' + request = response.request + try: + data = response.json() + except json.decoder.JSONDecodeError as e: + logging.critical(f'error decoding URL {request.url}: {e.args[0]}') + return {} + for result in parse_cai_results(data, 'cai-compute', method='list'): + resource = {} + resource_data = result['resource'] + resource_name = NAMES[resource_data['discoveryName']] + parent = resource_data['parent'] + if not _set_parent(resource, parent, resources): + logging.info(f'{result["name"]} outside perimeter') + continue + extend_func = globals().get(f'_handle_{resource_name}') + if not callable(extend_func): + raise SystemExit(f'specialized function missing for {resource_name}') + try: + extend_func(resource, resource_data['data']) + except Skip: + continue + resources[resource_name][resource['self_link']] = resource + page_token = data.get('nextPageToken') + if page_token: + logging.info('requesting next page') + url = _url(resources) + yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None) def _handle_networks(resource, data): @@ -129,6 +155,11 @@ def _handle_routers(resource, data): resource['region'] = data['region'].split('/')[-1] +def _self_link(s): + 'Remove initial part from self links.' + return s.replace('https://www.googleapis.com/compute/v1/', '') + + def _set_parent(resource, parent, resources): 'Extract and set resource parent.' parent_type, parent_id = parent.split('/')[-2:] @@ -157,7 +188,7 @@ def _url(resources): return CAI_URL.format(organization=organization, asset_types=asset_types) -@register(Phase.INIT, Step.START) +@register_init() def init(resources): 'Prepare the shared datastructures for asset types managed here.' for name in TYPES: @@ -165,40 +196,8 @@ def init(resources): resources[name] = {} -@register(Phase.DISCOVER, Step.START, Level.PRIMARY, 10) +@register_discovery(_handle_discovery, Level.PRIMARY, 10) def start_discovery(resources): 'Start discovery by returning the asset list URL for asset types.' logging.info('discovery compute start') yield HTTPRequest(_url(resources), {}, None) - - -@register(Phase.DISCOVER, Step.END) -def end_discovery(resources, response): - 'Process discovery data.' - request = response.request - try: - data = response.json() - except json.decoder.JSONDecodeError as e: - logging.critical(f'error decoding URL {request.url}: {e.args[0]}') - return {} - for result in parse_cai_results(data, 'cai-compute', method='list'): - resource = {} - resource_data = result['resource'] - resource_name = NAMES[resource_data['discoveryName']] - parent = resource_data['parent'] - if not _set_parent(resource, parent, resources): - logging.info(f'{result["name"]} outside perimeter') - continue - extend_func = globals().get(f'_handle_{resource_name}') - if not callable(extend_func): - raise SystemExit(f'specialized function missing for {resource_name}') - try: - extend_func(resource, resource_data['data']) - except Skip: - continue - resources[resource_name][resource['self_link']] = resource - page_token = data.get('nextPageToken') - if page_token: - logging.info('requesting next page') - url = _url(resources) - yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index c0e82778e5..acfdbdaa1f 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -15,7 +15,7 @@ import json import logging -from . import Level, Phase, HTTPRequest, Step, register +from . import HTTPRequest, Level, register_init, register_discovery from .utils import parse_cai_page_token, parse_cai_results NAME = 'projects' @@ -27,24 +27,7 @@ f'?assetTypes=cloudresourcemanager.googleapis.com%2FProject&pageSize=500') -@register(Phase.INIT, Step.START) -def init(resources): - if NAME not in resources: - resources[NAME] = {} - if 'project:numbers' not in resources: - resources['projects:number'] = {} - - -@register(Phase.DISCOVER, Step.START, Level.CORE, 0) -def start_discovery(resources): - logging.info('discovery projects start') - for resource_type in (NAME, 'folders'): - for k in resources.get(resource_type, []): - yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) - - -@register(Phase.DISCOVER, Step.END) -def end_discovery(resources, response): +def _handle_discovery(resources, response): request = response.request try: data = response.json() @@ -59,3 +42,19 @@ def end_discovery(resources, response): next_url = parse_cai_page_token(data, request.url) if next_url: yield HTTPRequest(next_url, {}, None) + + +@register_init() +def init(resources): + if NAME not in resources: + resources[NAME] = {} + if 'project:numbers' not in resources: + resources['projects:number'] = {} + + +@register_discovery(_handle_discovery, Level.CORE, 0) +def start_discovery(resources): + logging.info('discovery projects start') + for resource_type in (NAME, 'folders'): + for k in resources.get(resource_type, []): + yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py index 1522df0b92..b488d32d15 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py @@ -14,7 +14,7 @@ import logging -from . import Level, Phase, PluginError, Step, register +from . import Level, register_init, register_discovery from .utils import dirty_mp_request, dirty_mp_response NAME = 'project-quota' @@ -23,20 +23,7 @@ API_REGION_URL = '/compute/v1/projects/{}/regions/{}' -@register(Phase.INIT, Step.START) -def init(resources): - if NAME not in resources: - resources[NAME] = {} - - -@register(Phase.DISCOVER, Step.START, Level.DERIVED, 0) -def start_discovery(resources): - yield dirty_mp_request( - [API_GLOBAL_URL.format(p) for p in resources['projects']]) - - -@register(Phase.DISCOVER, Step.END) -def end_discovery(resources, response): +def _handle_discovery(resources, response): content_type = response.headers['content-type'] for part in dirty_mp_response(content_type, response.content): kind = part.get('kind') @@ -53,4 +40,16 @@ def end_discovery(resources, response): region = self_link[-1] project_quota = resources[NAME].setdefault(project_id, {}) project_quota[region] = quota - yield \ No newline at end of file + yield + + +@register_init() +def init(resources): + if NAME not in resources: + resources[NAME] = {} + + +@register_discovery(_handle_discovery, Level.DERIVED, 0) +def start_discovery(resources): + yield dirty_mp_request( + [API_GLOBAL_URL.format(p) for p in resources['projects']]) From d1c9a8afe82afe87a5bcbac8f94eacf73cdbf74e Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 15:39:58 +0100 Subject: [PATCH 12/82] streamline --- .../network-dashboard/cf/main.py | 16 ++++++++++------ .../network-dashboard/cf/plugins/__init__.py | 9 ++++++--- .../cf/plugins/discover-cai-compute.py | 13 ++++++++----- .../cf/plugins/discover-cai-projects.py | 10 +++++++--- ...roject-quota.py => discover-compute-quota.py} | 8 ++++++-- 5 files changed, 37 insertions(+), 19 deletions(-) rename blueprints/cloud-operations/network-dashboard/cf/plugins/{discover-compute-project-quota.py => discover-compute-quota.py} (90%) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 757ab14558..137556cc47 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -24,6 +24,7 @@ from google.auth.transport.requests import AuthorizedSession HTTP = AuthorizedSession(google.auth.default()[0]) +LOGGER = logging.getLogger('net-dash') Q_COLLECTION = collections.deque() RESOURCES = {} @@ -31,16 +32,17 @@ def do_discovery(): + LOGGER.info('discovery start') for plugin in plugins.get_discovery_plugins(): - logging.info(f'discovery {plugin.name}') requests = collections.deque(plugin.func(RESOURCES)) + LOGGER.info(f'discovery {plugin.name} ({len(requests)})') while requests: request = requests.popleft() response = fetch(request) for next_request in plugin.handler(RESOURCES, response): if not next_request: continue - logging.info(f'next request {next_request}') + LOGGER.info(f'discovery {plugin.name} (+1)') requests.append(next_request) @@ -57,7 +59,7 @@ def do_init(organization, folder, project): def fetch(request): # try - logging.info(f'fetch {request.url}') + LOGGER.info(f'fetch {request.url}') if not request.data: response = HTTP.get(request.url, headers=request.headers) else: @@ -65,7 +67,7 @@ def fetch(request): data=request.data) if response.status_code != 200: # TODO: handle this - logging.critical( + LOGGER.critical( f'response code {response.status_code} for URL {request.url}') print(request.url) print(request.headers) @@ -91,8 +93,10 @@ def main(organization=None, op_project=None, project=None, folder=None): do_discovery() - import icecream - icecream.ic(RESOURCES) + LOGGER.info({k: len(v) for k, v in RESOURCES.items()}) + + # import icecream + # icecream.ic(RESOURCES) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index d867214fa1..de13f898b7 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -15,9 +15,9 @@ import collections import enum import importlib -import itertools import pathlib import pkgutil +import types __all__ = [ 'HTTPRequest', 'Level', 'PluginError', 'Resource', 'get_discovery_plugins', @@ -58,8 +58,11 @@ def outer(func): return outer -def register_init(): - # TODO: make decorator work without args +def register_init(*args): + + if args and type(args[0]) == types.FunctionType: + _PLUGINS_INIT.append(Plugin(args[0], args[0].__module__)) + return def outer(func): _PLUGINS_INIT.append(Plugin(func, func.__module__)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 01f300ad79..b1454ea042 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -24,6 +24,7 @@ CAI_URL = ('https://content-cloudasset.googleapis.com/v1' '/organizations/{organization}/assets' '?contentType=RESOURCE&{asset_types}&pageSize=500') +LOGGER = logging.getLogger('net-dash.discovery.cai-compute') TYPES = { 'networks': 'Network', 'subnetworks': 'Subnetwork', @@ -43,10 +44,11 @@ class Skip(Exception): def _handle_discovery(resources, response): 'Process discovery data.' request = response.request + LOGGER.info('discovery handle request') try: data = response.json() except json.decoder.JSONDecodeError as e: - logging.critical(f'error decoding URL {request.url}: {e.args[0]}') + LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') return {} for result in parse_cai_results(data, 'cai-compute', method='list'): resource = {} @@ -54,7 +56,7 @@ def _handle_discovery(resources, response): resource_name = NAMES[resource_data['discoveryName']] parent = resource_data['parent'] if not _set_parent(resource, parent, resources): - logging.info(f'{result["name"]} outside perimeter') + LOGGER.info(f'{result["name"]} outside perimeter') continue extend_func = globals().get(f'_handle_{resource_name}') if not callable(extend_func): @@ -66,7 +68,7 @@ def _handle_discovery(resources, response): resources[resource_name][resource['self_link']] = resource page_token = data.get('nextPageToken') if page_token: - logging.info('requesting next page') + LOGGER.info('requesting next page') url = _url(resources) yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None) @@ -188,9 +190,10 @@ def _url(resources): return CAI_URL.format(organization=organization, asset_types=asset_types) -@register_init() +@register_init def init(resources): 'Prepare the shared datastructures for asset types managed here.' + LOGGER.info('init') for name in TYPES: if name not in resources: resources[name] = {} @@ -199,5 +202,5 @@ def init(resources): @register_discovery(_handle_discovery, Level.PRIMARY, 10) def start_discovery(resources): 'Start discovery by returning the asset list URL for asset types.' - logging.info('discovery compute start') + LOGGER.info('discovery start') yield HTTPRequest(_url(resources), {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index acfdbdaa1f..82aa030b7e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -18,6 +18,7 @@ from . import HTTPRequest, Level, register_init, register_discovery from .utils import parse_cai_page_token, parse_cai_results +LOGGER = logging.getLogger('net-dash.discovery.cai-projects') NAME = 'projects' TYPE = 'cloudresourcemanager.googleapis.com/Project' @@ -28,11 +29,12 @@ def _handle_discovery(resources, response): + LOGGER.info('discovery handle request') request = response.request try: data = response.json() except json.decoder.JSONDecodeError as e: - logging.critical(f'error decoding URL {request.url}: {e.args[0]}') + LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') return {} for result in parse_cai_results(data, NAME, TYPE): number = result['project'].split('/')[1] @@ -41,11 +43,13 @@ def _handle_discovery(resources, response): resources['projects:number'][number] = project_id next_url = parse_cai_page_token(data, request.url) if next_url: + LOGGER.info('discovery next url') yield HTTPRequest(next_url, {}, None) -@register_init() +@register_init def init(resources): + LOGGER.info('init') if NAME not in resources: resources[NAME] = {} if 'project:numbers' not in resources: @@ -54,7 +58,7 @@ def init(resources): @register_discovery(_handle_discovery, Level.CORE, 0) def start_discovery(resources): - logging.info('discovery projects start') + LOGGER.info('discovery start') for resource_type in (NAME, 'folders'): for k in resources.get(resource_type, []): yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py similarity index 90% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index b488d32d15..81a9d85411 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-project-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -17,13 +17,15 @@ from . import Level, register_init, register_discovery from .utils import dirty_mp_request, dirty_mp_response -NAME = 'project-quota' +LOGGER = logging.getLogger('net-dash.discovery.compute-quota') +NAME = 'quota' API_GLOBAL_URL = '/compute/v1/projects/{}' API_REGION_URL = '/compute/v1/projects/{}/regions/{}' def _handle_discovery(resources, response): + LOGGER.info('discovery handle request') content_type = response.headers['content-type'] for part in dirty_mp_response(content_type, response.content): kind = part.get('kind') @@ -43,13 +45,15 @@ def _handle_discovery(resources, response): yield -@register_init() +@register_init def init(resources): + LOGGER.info('init') if NAME not in resources: resources[NAME] = {} @register_discovery(_handle_discovery, Level.DERIVED, 0) def start_discovery(resources): + LOGGER.info('discovery start') yield dirty_mp_request( [API_GLOBAL_URL.format(p) for p in resources['projects']]) From 10a4b1fdaac614e08f4479dd381bd8a34288ac6f Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 16:17:19 +0100 Subject: [PATCH 13/82] dynamic routes --- .../plugins/discover-compute-routerstatus.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py new file mode 100644 index 0000000000..e96b17a5bd --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -0,0 +1,70 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from . import Level, register_init, register_discovery +from .utils import dirty_mp_request, dirty_mp_response + +LOGGER = logging.getLogger('net-dash.discovery.compute-routes-dynamic') +NAME = 'routes-dynamic' + +API_URL = '/compute/v1/projects/{}/regions/{}/routers/{}/getRouterStatus' + + +def _handle_discovery(resources, response): + LOGGER.info('discovery handle request') + content_type = response.headers['content-type'] + routers = [r for r in resources['routers'].values()] + for i, part in enumerate(dirty_mp_response(content_type, response.content)): + router = routers[i] + result = part.get('result') + if not result: + LOGGER.info(f'skipping router {router["self_link"]}, no result') + continue + bgp_peer_status = result.get('bgpPeerStatus') + if not bgp_peer_status: + LOGGER.info(f'skipping router {router["self_link"]}, no bgp peer status') + continue + network = result.get('network') + if not network: + LOGGER.info(f'skipping router {router["self_link"]}, no bgp peer status') + continue + if not network.endswith(router['network']): + LOGGER.warn( + f'router network mismatch: got {network} expected {router["network"]}' + ) + continue + num_learned_routes = sum( + int(p.get('numLearnedRoutes', 0)) for p in bgp_peer_status) + resources[NAME][router['network']] = resources[NAME].get( + router['network'], 0) + num_learned_routes + yield + + +@register_init +def init(resources): + LOGGER.info('init') + if NAME not in resources: + resources[NAME] = {} + + +@register_discovery(_handle_discovery, Level.DERIVED) +def start_discovery(resources): + LOGGER.info('discovery start') + urls = [ + API_URL.format(r['project_id'], r['region'], r['name']) + for r in resources['routers'].values() + ] + yield dirty_mp_request(urls) From 1274780abcfb584ed1aab204f8203d7e19a07f90 Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 16:17:58 +0100 Subject: [PATCH 14/82] dynamic routes --- .../network-dashboard/cf/plugins/discover-compute-quota.py | 1 + .../cf/plugins/discover-compute-routerstatus.py | 1 + 2 files changed, 2 insertions(+) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 81a9d85411..cc2b541695 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -55,5 +55,6 @@ def init(resources): @register_discovery(_handle_discovery, Level.DERIVED, 0) def start_discovery(resources): LOGGER.info('discovery start') + # TODO: split in batches yield dirty_mp_request( [API_GLOBAL_URL.format(p) for p in resources['projects']]) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index e96b17a5bd..806d282c94 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -63,6 +63,7 @@ def init(resources): @register_discovery(_handle_discovery, Level.DERIVED) def start_discovery(resources): LOGGER.info('discovery start') + # TODO: split in batches urls = [ API_URL.format(r['project_id'], r['region'], r['name']) for r in resources['routers'].values() From 9c48e444da1207615d79c52998c8e9c9eeb935cd Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 20:36:33 +0100 Subject: [PATCH 15/82] forwarding rules and addresses --- .../network-dashboard/cf/plugins/__init__.py | 39 +++++++- .../cf/plugins/discover-cai-compute.py | 95 +++++++++++++------ .../cf/plugins/series-networks.py | 13 +++ 3 files changed, 111 insertions(+), 36 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index de13f898b7..002602f640 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -24,8 +24,10 @@ 'get_init_plugins', 'register_discovery', 'register_init' ] -_PLUGINS_INIT = [] +_PLUGINS_SERIES = [] _PLUGINS_DISCOVERY = [] +_PLUGINS_INIT = [] +_PLUGINS_SERIES = [] HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data') Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') @@ -39,33 +41,60 @@ class PluginError(Exception): def get_discovery_plugins(): + 'Return discovery plugins.' for p in _PLUGINS_DISCOVERY: yield p def get_init_plugins(): + 'Return init plugins.' for p in _PLUGINS_INIT: yield p +def get_series_plugins(): + 'Return metrics plugins.' + for p in _PLUGINS_SERIES: + yield p + + +def _register(collection, func, *args): + 'Derive plugin name from function and add to its collection.' + name = f'{func.__module__}.{func.__name__}' + collection.append(Plugin(func, name, *args)) + + def register_discovery(handler_func, level=Level.PRIMARY, priority=99): + 'Register plugins that discover data.' def outer(func): - _PLUGINS_DISCOVERY.append( - Plugin(func, func.__module__, level, priority, handler_func)) + _register(_PLUGINS_DISCOVERY, func, level, priority, handler_func) return func return outer def register_init(*args): + 'Register plugins that prepare the shared data structure.' + if args and type(args[0]) == types.FunctionType: + _register(_PLUGINS_INIT, args[0]) + return + + def outer(func): + _register(_PLUGINS_INIT, func) + return func + + return outer + +def register_series(*args): + 'Register plugins that derive metrics series from data.' if args and type(args[0]) == types.FunctionType: - _PLUGINS_INIT.append(Plugin(args[0], args[0].__module__)) + _register(_PLUGINS_SERIES, args[0]) return def outer(func): - _PLUGINS_INIT.append(Plugin(func, func.__module__)) + _register(_PLUGINS_SERIES, func) return func return outer diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index b1454ea042..22198e9e9e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -26,13 +26,15 @@ '?contentType=RESOURCE&{asset_types}&pageSize=500') LOGGER = logging.getLogger('net-dash.discovery.cai-compute') TYPES = { - 'networks': 'Network', - 'subnetworks': 'Subnetwork', - 'firewalls': 'Firewall', + 'addresses': 'Address', 'firewall_policies': 'FirewallPolicy', + 'firewalls': 'Firewall', + 'forwarding_rules': 'ForwardingRule', 'instances': 'Instance', + 'networks': 'Network', + 'subnetworks': 'Subnetwork', + 'routers': 'Router', 'routes': 'Route', - 'routers': 'Router' } NAMES = {v: k for k, v in TYPES.items()} @@ -73,34 +75,27 @@ def _handle_discovery(resources, response): yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None) -def _handle_networks(resource, data): - 'Handle network type resource data.' +def _handle_addresses(resource, data): + 'Handle address type resource data.' resource['id'] = data['id'] resource['name'] = data['name'] resource['self_link'] = _self_link(data['selfLink']) - resource['peerings'] = [] - for p in data.get('peerings', []): - if p['state'] != 'ACTIVE': - continue - resource['peerings'].append({'name': p['name'], 'network': p['network']}) - resource['subnets'] = [_self_link(s) for s in data.get('subnetworks', [])] + resource['network'] = _self_link( + data['network']) if 'network' in data else None + resource['subnetwork'] = _self_link( + data['subnetwork']) if 'subnetwork' in data else None + resource['address'] = data['address'] + resource['internal'] = data.get('addressType') == 'INTERNAL' + resource['purpose'] = data.get('purpose') -def _handle_subnetworks(resource, data): - 'Handle subnetwork type resource data.' +def _handle_firewall_policies(resource, data): + 'Handle firewall policy type resource data.' resource['id'] = data['id'] resource['name'] = data['name'] resource['self_link'] = _self_link(data['selfLink']) - resource['cidr_range'] = data['ipCidrRange'] - resource['network'] = _self_link(data['network']) - resource['purpose'] = data.get('purpose') - resource['region'] = data['region'] - resource['secondary_ranges'] = [] - for s in data.get('secondaryIpRanges', []): - resource['secondary_ranges'].append({ - 'name': s['rangeName'], - 'cidr_range': s['ipCidrRange'] - }) + resource['num_rules'] = len(data.get('rules', [])) + resource['num_tuples'] = data.get('ruleTupleCount', 0) def _handle_firewalls(resource, data): @@ -111,13 +106,21 @@ def _handle_firewalls(resource, data): resource['network'] = _self_link(data['network']) -def _handle_firewall_policies(resource, data): - 'Handle firewall policy type resource data.' +def _handle_forwarding_rules(resource, data): + 'Handle forwarding_rules type resource data.' + from icecream import ic resource['id'] = data['id'] resource['name'] = data['name'] resource['self_link'] = _self_link(data['selfLink']) - resource['num_rules'] = len(data.get('rules', [])) - resource['num_tuples'] = data.get('ruleTupleCount', 0) + resource['region'] = data['region'].split( + '/')[-1] if 'region' in data else None + resource['load_balancing_scheme'] = data['loadBalancingScheme'] + resource['address'] = data['IPAddress'] if 'IPAddress' in data else None + resource['network'] = _self_link( + data['network']) if 'network' in data else None + resource['subnetwork'] = _self_link( + data['subnetwork']) if 'subnetwork' in data else None + resource['psc'] = True if data.get('pscConnectionStatus') else False def _handle_instances(resource, data): @@ -136,6 +139,28 @@ def _handle_instances(resource, data): }) +def _handle_networks(resource, data): + 'Handle network type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['peerings'] = [] + for p in data.get('peerings', []): + if p['state'] != 'ACTIVE': + continue + resource['peerings'].append({'name': p['name'], 'network': p['network']}) + resource['subnets'] = [_self_link(s) for s in data.get('subnetworks', [])] + + +def _handle_routers(resource, data): + 'Handle router type resource data.' + resource['id'] = data['id'] + resource['name'] = data['name'] + resource['self_link'] = _self_link(data['selfLink']) + resource['network'] = _self_link(data['network']) + resource['region'] = data['region'].split('/')[-1] + + def _handle_routes(resource, data): 'Handle route type resource data.' resource['id'] = data['id'] @@ -148,13 +173,21 @@ def _handle_routes(resource, data): resource['next_hope_type'] = hop[0] -def _handle_routers(resource, data): - 'Handle router type resource data.' +def _handle_subnetworks(resource, data): + 'Handle subnetwork type resource data.' resource['id'] = data['id'] resource['name'] = data['name'] resource['self_link'] = _self_link(data['selfLink']) + resource['cidr_range'] = data['ipCidrRange'] resource['network'] = _self_link(data['network']) - resource['region'] = data['region'].split('/')[-1] + resource['purpose'] = data.get('purpose') + resource['region'] = data['region'] + resource['secondary_ranges'] = [] + for s in data.get('secondaryIpRanges', []): + resource['secondary_ranges'].append({ + 'name': s['rangeName'], + 'cidr_range': s['ipCidrRange'] + }) def _self_link(s): diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py new file mode 100644 index 0000000000..6d6d1266c3 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 5321f781b250f7191d56544708ab07f5275cd349 Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 17 Nov 2022 20:47:31 +0100 Subject: [PATCH 16/82] batch requests --- .../cf/plugins/discover-compute-quota.py | 10 ++-- .../plugins/discover-compute-routerstatus.py | 8 +-- .../network-dashboard/cf/plugins/utils.py | 51 ++++++++++++------- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index cc2b541695..127e034ac5 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -15,7 +15,7 @@ import logging from . import Level, register_init, register_discovery -from .utils import dirty_mp_request, dirty_mp_response +from .utils import batched, dirty_mp_request, dirty_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-quota') NAME = 'quota' @@ -55,6 +55,8 @@ def init(resources): @register_discovery(_handle_discovery, Level.DERIVED, 0) def start_discovery(resources): LOGGER.info('discovery start') - # TODO: split in batches - yield dirty_mp_request( - [API_GLOBAL_URL.format(p) for p in resources['projects']]) + urls = [API_GLOBAL_URL.format(p) for p in resources['projects']] + if not urls: + return + for batch in batched(urls, 10): + yield dirty_mp_request(batch) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index 806d282c94..f56fb98d84 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -15,7 +15,7 @@ import logging from . import Level, register_init, register_discovery -from .utils import dirty_mp_request, dirty_mp_response +from .utils import batched, dirty_mp_request, dirty_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-routes-dynamic') NAME = 'routes-dynamic' @@ -63,9 +63,11 @@ def init(resources): @register_discovery(_handle_discovery, Level.DERIVED) def start_discovery(resources): LOGGER.info('discovery start') - # TODO: split in batches urls = [ API_URL.format(r['project_id'], r['region'], r['name']) for r in resources['routers'].values() ] - yield dirty_mp_request(urls) + if not urls: + return + for batch in batched(urls, 10): + yield dirty_mp_request(batch) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index b60394a2ae..e0b541c72b 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import json import logging import re @@ -35,27 +36,18 @@ RE_URL = re.compile(r'nextPageToken=[^&]+&?') -def parse_cai_results(data, name, resource_type=None, method='search'): - results = data.get('results' if method == 'search' else 'assets') - if not results: - logging.info(f'no results for {name}') - return - for result in results: - if resource_type and result['assetType'] != resource_type: - logging.warn(f'result for wrong type {result["assetType"]}') - continue - yield result - - -def parse_cai_page_token(data, url): - page_token = data.get('nextPageToken') - if page_token: - logging.info(f'page token {page_token}') - if page_token: - return RE_URL.sub(f'pageToken={page_token}&', url) +def batched(iterable, n): + 'Batch data into lists of length n. The last batch may be shorter.' + # batched('ABCDEFG', 3) --> ABC DEF G + if n < 1: + raise ValueError('n must be at least one') + it = iter(iterable) + while (batch := list(itertools.islice(it, n))): + yield batch def dirty_mp_request(urls, boundary='1234567890'): + 'Bundle urls into a single multipart mixed batched request.' boundary = f'--{boundary}' data = [boundary] for url in urls: @@ -67,6 +59,7 @@ def dirty_mp_request(urls, boundary='1234567890'): def dirty_mp_response(content_type, content): + 'Parse multipart mixed response and return individual parts.' try: _, boundary = content_type.split('=') except ValueError: @@ -83,3 +76,25 @@ def dirty_mp_response(content_type, content): except ValueError: raise PluginError('cannot parse MIME part') yield json.loads(body) + + +def parse_cai_results(data, name, resource_type=None, method='search'): + 'Preliminary parsing of CAI asset result.' + results = data.get('results' if method == 'search' else 'assets') + if not results: + logging.info(f'no results for {name}') + return + for result in results: + if resource_type and result['assetType'] != resource_type: + logging.warn(f'result for wrong type {result["assetType"]}') + continue + yield result + + +def parse_cai_page_token(data, url): + 'Detect next page token in result and return next page URL.' + page_token = data.get('nextPageToken') + if page_token: + logging.info(f'page token {page_token}') + if page_token: + return RE_URL.sub(f'pageToken={page_token}&', url) From fb2efabed0466677312c84ba8f9086e926ac24ba Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 19 Nov 2022 16:40:17 +0100 Subject: [PATCH 17/82] metrics --- .../network-dashboard/cf/main.py | 17 +- .../cf/plugins/discover-cai-compute.py | 190 ++++++++---------- .../cf/plugins/discover-cai-projects.py | 4 +- .../cf/plugins/discover-metrics.py | 64 ++++++ .../network-dashboard/cf/plugins/utils.py | 2 +- 5 files changed, 157 insertions(+), 120 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 137556cc47..5cdf1e5b9f 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -46,13 +46,14 @@ def do_discovery(): requests.append(next_request) -def do_init(organization, folder, project): - if organization: - RESOURCES['organization'] = {'id': organization} +def do_init(organization, folder, project, op_project): + RESOURCES['organization'] = str(organization) + RESOURCES['monitoring_project'] = op_project if folder: RESOURCES['folders'] = {f: {} for f in folder} if project: RESOURCES['projects'] = {p: {} for p in project} + for plugin in plugins.get_init_plugins(): plugin.func(RESOURCES) @@ -69,11 +70,6 @@ def fetch(request): # TODO: handle this LOGGER.critical( f'response code {response.status_code} for URL {request.url}') - print(request.url) - print(request.headers) - print(request.data) - print(response.headers) - print(response.content) return response @@ -89,11 +85,12 @@ def fetch(request): def main(organization=None, op_project=None, project=None, folder=None): logging.basicConfig(level=logging.INFO) - do_init(organization, folder, project) + do_init(organization, folder, project, op_project) do_discovery() - LOGGER.info({k: len(v) for k, v in RESOURCES.items()}) + LOGGER.info( + {k: len(v) for k, v in RESOURCES.items() if not isinstance(v, str)}) # import icecream # icecream.ic(RESOURCES) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 22198e9e9e..267f56f651 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -39,10 +39,6 @@ NAMES = {v: k for k, v in TYPES.items()} -class Skip(Exception): - pass - - def _handle_discovery(resources, response): 'Process discovery data.' request = response.request @@ -53,21 +49,7 @@ def _handle_discovery(resources, response): LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') return {} for result in parse_cai_results(data, 'cai-compute', method='list'): - resource = {} - resource_data = result['resource'] - resource_name = NAMES[resource_data['discoveryName']] - parent = resource_data['parent'] - if not _set_parent(resource, parent, resources): - LOGGER.info(f'{result["name"]} outside perimeter') - continue - extend_func = globals().get(f'_handle_{resource_name}') - if not callable(extend_func): - raise SystemExit(f'specialized function missing for {resource_name}') - try: - extend_func(resource, resource_data['data']) - except Skip: - continue - resources[resource_name][resource['self_link']] = resource + _handle_resource(resources, result['resource']) page_token = data.get('nextPageToken') if page_token: LOGGER.info('requesting next page') @@ -75,119 +57,118 @@ def _handle_discovery(resources, response): yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None) +def _handle_resource(resources, data): + attrs = data['data'] + resource_name = NAMES[data['discoveryName']] + resource = { + 'id': attrs['id'], + 'name': attrs['name'], + 'self_link': _self_link(attrs['selfLink']) + } + parent_data = _get_parent(data['parent'], resources) + if not parent_data: + LOGGER.info(f'{resource["self_link"]} outside perimeter') + return + resource.update(parent_data) + func = globals().get(f'_handle_{resource_name}') + if not callable(func): + raise SystemExit(f'specialized function missing for {resource_name}') + extra_attrs = func(resource, attrs) + if not extra_attrs: + return + resource.update(extra_attrs) + resources[resource_name][resource['self_link']] = resource + + def _handle_addresses(resource, data): 'Handle address type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['network'] = _self_link( - data['network']) if 'network' in data else None - resource['subnetwork'] = _self_link( - data['subnetwork']) if 'subnetwork' in data else None - resource['address'] = data['address'] - resource['internal'] = data.get('addressType') == 'INTERNAL' - resource['purpose'] = data.get('purpose') + network = data.get('network') + subnet = data.get('subnetwork') + return { + 'address': data['address'], + 'purpose': data.get('purpose'), + 'internal': data.get('addressType') == 'INTERNAL', + 'network': None if not network else _self_link(network), + 'subnet': None if not subnet else _self_link(subnet) + } def _handle_firewall_policies(resource, data): 'Handle firewall policy type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['num_rules'] = len(data.get('rules', [])) - resource['num_tuples'] = data.get('ruleTupleCount', 0) + return { + 'num_rules': len(data.get('rules', [])), + 'num_tuples': data.get('ruleTupleCount', 0) + } def _handle_firewalls(resource, data): 'Handle firewall type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['network'] = _self_link(data['network']) + return {'network': _self_link(data['network'])} def _handle_forwarding_rules(resource, data): 'Handle forwarding_rules type resource data.' - from icecream import ic - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['region'] = data['region'].split( - '/')[-1] if 'region' in data else None - resource['load_balancing_scheme'] = data['loadBalancingScheme'] - resource['address'] = data['IPAddress'] if 'IPAddress' in data else None - resource['network'] = _self_link( - data['network']) if 'network' in data else None - resource['subnetwork'] = _self_link( - data['subnetwork']) if 'subnetwork' in data else None - resource['psc'] = True if data.get('pscConnectionStatus') else False + network = data.get('network') + region = data.get('region') + subnet = data.get('subnetwork') + return { + 'address': data.get('IPAddress'), + 'load_balancing_scheme': data['loadBalancingScheme'], + 'network': None if not network else _self_link(network), + 'region': None if not region else region.split('/')[-1], + 'subnet': None if not subnet else _self_link(subnet) + } def _handle_instances(resource, data): 'Handle instance type resource data.' if data['status'] != 'RUNNING': - raise Skip() - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['zone'] = data['zone'] - resource['networks'] = [] - for i in data.get('networkInterfaces', []): - resource['networks'].append({ - 'network': _self_link(i['network']), - 'subnet': _self_link(i['subnetwork']) - }) + return + networks = [{ + 'network': _self_link(i['network']), + 'subnet': _self_link(i['subnetwork']) + } for i in data.get('networkInterfaces', [])] + return {'zone': data['zone'], 'networks': networks} def _handle_networks(resource, data): 'Handle network type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['peerings'] = [] - for p in data.get('peerings', []): - if p['state'] != 'ACTIVE': - continue - resource['peerings'].append({'name': p['name'], 'network': p['network']}) - resource['subnets'] = [_self_link(s) for s in data.get('subnetworks', [])] + peerings = [{ + 'name': p['name'], + 'network': _self_link(p['network']) + } for p in data.get('peerings', []) if p['state'] == 'ACTIVE'] + subnets = [_self_link(s) for s in data.get('subnetworks', [])] + return {'peerings': peerings, 'subnets': subnets} def _handle_routers(resource, data): 'Handle router type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['network'] = _self_link(data['network']) - resource['region'] = data['region'].split('/')[-1] + return { + 'network': _self_link(data['network']), + 'region': data['region'].split('/')[-1] + } def _handle_routes(resource, data): 'Handle route type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['network'] = _self_link(data['network']) hop = [ a.replace('nextHop', '').lower() for a in data if a.startswith('nextHop') ] - resource['next_hope_type'] = hop[0] + return {'next_hop_type': hop[0], 'network': _self_link(data['network'])} def _handle_subnetworks(resource, data): 'Handle subnetwork type resource data.' - resource['id'] = data['id'] - resource['name'] = data['name'] - resource['self_link'] = _self_link(data['selfLink']) - resource['cidr_range'] = data['ipCidrRange'] - resource['network'] = _self_link(data['network']) - resource['purpose'] = data.get('purpose') - resource['region'] = data['region'] - resource['secondary_ranges'] = [] - for s in data.get('secondaryIpRanges', []): - resource['secondary_ranges'].append({ - 'name': s['rangeName'], - 'cidr_range': s['ipCidrRange'] - }) + secondary_ranges = [{ + 'name': s['rangeName'], + 'cidr_range': s['ipCidrRange'] + } for s in data.get('secondaryIpRanges', [])] + return { + 'cidr_range': data['ipCidrRange'], + 'network': _self_link(data['network']), + 'purpose': data.get('purpose'), + 'region': data['region'] + } def _self_link(s): @@ -195,28 +176,23 @@ def _self_link(s): return s.replace('https://www.googleapis.com/compute/v1/', '') -def _set_parent(resource, parent, resources): - 'Extract and set resource parent.' +def _get_parent(parent, resources): + 'Extract and return resource parent.' parent_type, parent_id = parent.split('/')[-2:] - update = None if parent_type == 'projects': project_id = resources['projects:number'].get(parent_id) if project_id: - update = {'project_id': project_id, 'project_number': parent_id} - elif parent_type == 'folders': + return {'project_id': project_id, 'project_number': parent_id} + if parent_type == 'folders': if int(parent_id) in resources['folders']: - update = {'parent': f'{parent_type}/{parent_id}'} - elif parent_type == 'organizations': - if resources['organization']['id'] == int(parent_id): - update = {'parent': f'{parent_type}/{parent_id}'} - if update: - resource.update(update) - return update is not None + return {'parent': f'{parent_type}/{parent_id}'} + if resources['organization'] == int(parent_id): + return {'parent': f'{parent_type}/{parent_id}'} def _url(resources): 'Return discovery URL' - organization = resources['organization']['id'] + organization = resources['organization'] asset_types = '&'.join( 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) for t in TYPES.values()) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index 82aa030b7e..8daeb047d4 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -16,7 +16,7 @@ import logging from . import HTTPRequest, Level, register_init, register_discovery -from .utils import parse_cai_page_token, parse_cai_results +from .utils import parse_page_token, parse_cai_results LOGGER = logging.getLogger('net-dash.discovery.cai-projects') NAME = 'projects' @@ -41,7 +41,7 @@ def _handle_discovery(resources, response): project_id = result['displayName'] resources[NAME][project_id] = {'number': number} resources['projects:number'][number] = project_id - next_url = parse_cai_page_token(data, request.url) + next_url = parse_page_token(data, request.url) if next_url: LOGGER.info('discovery next url') yield HTTPRequest(next_url, {}, None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py new file mode 100644 index 0000000000..c42d3c3c29 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py @@ -0,0 +1,64 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import urllib.parse + +from . import HTTPRequest, Level, register_init, register_discovery +from .utils import parse_page_token, parse_cai_results + +LOGGER = logging.getLogger('net-dash.discovery.metrics') +NAME = 'metrics' + +URL = ( + 'https://content-monitoring.googleapis.com/v3/projects' + '/{}/metricDescriptors' + '?filter=metric.type%3Dstarts_with(%22custom.googleapis.com%2Fnetmon%2F%22)' + '&pageSize=500') + + +def _handle_discovery(resources, response): + LOGGER.info('discovery handle request') + request = response.request + try: + data = response.json() + except json.decoder.JSONDecodeError as e: + LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') + return {} + descriptors = data.get('metricDescriptors') + if not descriptors: + LOGGER.info('no descriptors found') + return + for d in descriptors: + resources['metrics'].append(d['type']) + next_url = parse_page_token(data, request.url) + if next_url: + LOGGER.info('discovery next url') + yield HTTPRequest(next_url, {}, None) + + +@register_init +def init(resources): + LOGGER.info('init') + if 'metrics' not in resources: + resources['metrics'] = [] + + +@register_discovery(_handle_discovery, Level.CORE, 0) +def start_discovery(resources): + LOGGER.info('discovery start') + yield HTTPRequest( + URL.format(urllib.parse.quote_plus(resources['monitoring_project'])), {}, + None) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index e0b541c72b..ad891a41c2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -91,7 +91,7 @@ def parse_cai_results(data, name, resource_type=None, method='search'): yield result -def parse_cai_page_token(data, url): +def parse_page_token(data, url): 'Detect next page token in result and return next page URL.' page_token = data.get('nextPageToken') if page_token: From 5cd622b20fca007f5a4f0feea2f7ffae59f4c3b3 Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 19 Nov 2022 19:30:35 +0100 Subject: [PATCH 18/82] notes --- .../network-dashboard/cf/NOTES.txt | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt b/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt index 1b8695c665..c981e2566d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt @@ -1,5 +1,72 @@ # Notes +- get projects + get_monitored_projects_list +- set monitoring interval + monitoring_interval +- read metrics and limits from yaml and create descriptors + metrics.create_metrics +- get project quota + limits.get_quota_project_limit +- get firewall rules from CAI + vpc_firewalls.get_firewalls_dict +- get firewall policies from CAI + firewall_policies.get_firewall_policies_dict +- get and store subnet metrics + subnets.get_subnets + - get subnets + get_all_subnets + - calculate subnet utilization + compute_subnet_utilization + - get instances + compute_subnet_utilization_vms + - get forwarding rules + compute_subnet_utilization_ilbs + - get addresses + compute_subnet_utilization_addresses + - get redis instances + compute_subnet_utilization_redis + - store metrics +- get instances + instances.get_gce_instance_dict +- get forwarding rules for L4 ILB + ilb_fwrules.get_forwarding_rules_dict +- get forwarding rules for L7 ILB + ilb_fwrules.get_forwarding_rules_dict +- get subnets and secondary ranges + networks.get_subnet_ranges_dict +- get static routes + routes.get_static_routes_dict +- get dynamic routes + routes.get_dynamic_routes + - get routers + routers.get_routers + - get networks + networks.get_networks + - get router status + get_routes_for_network + get_routes_for_router +- calculate and store firewall rule metrics + vpc_firewalls.get_firewalls_data +- calculate and store firewall policy metrics + firewall_policies.get_firewal_policies_data +- calculate and store instance per network metrics + instances.get_gce_instances_data +- calculate and store L4 forwarding rule metrics + ilb_fwrules.get_forwarding_rules_data +- calculate and store L7 forwarding rule metrics + ilb_fwrules.get_forwarding_rules_data +- calculate and store static routes metrics + routes.get_static_routes_data +- calculate and store peering metrics + peerings.get_vpc_peering_data +- calculate and store peering group metrics + metrics.get_pgg_data + routes.get_routes_ppg +- write buffered timeseries + metrics.flush_series_buffer + + ## Inputs direct inputs From dea4a13ab0d999ae2fddc09c44968be7654a9aef Mon Sep 17 00:00:00 2001 From: Ludo Date: Sun, 20 Nov 2022 13:02:48 +0100 Subject: [PATCH 19/82] notes --- .../network-dashboard/cf/NOTES.md | 133 ++++++++++++++++++ .../network-dashboard/cf/NOTES.txt | 133 ------------------ 2 files changed, 133 insertions(+), 133 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/NOTES.md delete mode 100644 blueprints/cloud-operations/network-dashboard/cf/NOTES.txt diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md new file mode 100644 index 0000000000..9526ed94ce --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -0,0 +1,133 @@ +# Notes + +- [x] get projects + `get_monitored_projects_list` +- [ ] set monitoring interval + `monitoring_interval` +- [ ] read metrics and limits from yaml and create descriptors + `metrics.create_metrics` +- [x] get project quota + `limits.get_quota_project_limit` +- [x] get firewall rules from CAI + `vpc_firewalls.get_firewalls_dict` +- [x] get firewall policies from CAI + `firewall_policies.get_firewall_policies_dict` +- [x] get instances + `instances.get_gce_instance_dict` +- [x] get forwarding rules for L4 ILB + `ilb_fwrules.get_forwarding_rules_dict` +- [x] get forwarding rules for L7 ILB + `ilb_fwrules.get_forwarding_rules_dict` +- [x] get subnets and secondary ranges + `networks.get_subnet_ranges_dict` +- [x] get static routes + `routes.get_static_routes_dict` +- [x] get dynamic routes + routes.get_dynamic_routes + - get routers + `routers.get_routers` + - get networks + `networks.get_networks` + - get router status + `get_routes_for_network` + `get_routes_for_router` +- [ ] get and store subnet metrics + `subnets.get_subnets` + - get subnets + `get_all_subnets` + - calculate subnet utilization + `compute_subnet_utilization` + - get instances + `compute_subnet_utilization_vms` + - get forwarding rules + `compute_subnet_utilization_ilbs` + - get addresses + `compute_subnet_utilization_addresses` + - get redis instances + `compute_subnet_utilization_redis` + - store metrics +- [ ]calculate and store firewall rule metrics + `vpc_firewalls.get_firewalls_data` +- [ ] calculate and store firewall policy metrics + `firewall_policies.get_firewal_policies_data` +- [ ] calculate and store instance per network metrics + `instances.get_gce_instances_data` +- [ ] calculate and store L4 forwarding rule metrics + `ilb_fwrules.get_forwarding_rules_data` +- [ ] calculate and store L7 forwarding rule metrics + `ilb_fwrules.get_forwarding_rules_data` +- [ ] calculate and store static routes metrics + `routes.get_static_routes_data` +- [ ] calculate and store peering metrics + `peerings.get_vpc_peering_data` +- [ ] calculate and store peering group metrics + `metrics.get_pgg_data` + `routes.get_routes_ppg` +- [ ] write buffered timeseries + `metrics.flush_series_buffer` + + +## Inputs + +direct inputs + +- organization id +- folders (monitored) +- projects (monitored) + +derived inputs + +- projects in folders via CAI +- networks +- subnets +- routers +- peerings +- quotas +- firewall rules +- firewall policies +- routes +- routers +- dynamic routes +- peerings +- instances + +resources + +- project quota +- firewall rules in org via CAI + - key: network +- firewall policies in org via CAI + - key: network +- networks in org via CAI +- subnets in org via CAI + - key: project, network? + - computed metrics: ip usage (used, total, utilization) + - computed metrics: secondary IP ranges +- instances + - key: network + - computed metrics: instance per network (usage, limit, utilization) +- forwarding rules in org via CAI + - key: network, type + - computed metrics: fwd rule per network per type (usage, limit, utilization) +- static routes in org via CAI + - computed metrics: routes per project (usage, limit, utilization) +- dynamic routes via routers + - computed metrics: routes per project (usage, limit, utilization) + + + +## Resources and data + +- projects + - quotas + + +## Metrics + + +## Clients + +- compute +- asset inventory +- momnitoring + diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt b/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt deleted file mode 100644 index c981e2566d..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.txt +++ /dev/null @@ -1,133 +0,0 @@ -# Notes - -- get projects - get_monitored_projects_list -- set monitoring interval - monitoring_interval -- read metrics and limits from yaml and create descriptors - metrics.create_metrics -- get project quota - limits.get_quota_project_limit -- get firewall rules from CAI - vpc_firewalls.get_firewalls_dict -- get firewall policies from CAI - firewall_policies.get_firewall_policies_dict -- get and store subnet metrics - subnets.get_subnets - - get subnets - get_all_subnets - - calculate subnet utilization - compute_subnet_utilization - - get instances - compute_subnet_utilization_vms - - get forwarding rules - compute_subnet_utilization_ilbs - - get addresses - compute_subnet_utilization_addresses - - get redis instances - compute_subnet_utilization_redis - - store metrics -- get instances - instances.get_gce_instance_dict -- get forwarding rules for L4 ILB - ilb_fwrules.get_forwarding_rules_dict -- get forwarding rules for L7 ILB - ilb_fwrules.get_forwarding_rules_dict -- get subnets and secondary ranges - networks.get_subnet_ranges_dict -- get static routes - routes.get_static_routes_dict -- get dynamic routes - routes.get_dynamic_routes - - get routers - routers.get_routers - - get networks - networks.get_networks - - get router status - get_routes_for_network - get_routes_for_router -- calculate and store firewall rule metrics - vpc_firewalls.get_firewalls_data -- calculate and store firewall policy metrics - firewall_policies.get_firewal_policies_data -- calculate and store instance per network metrics - instances.get_gce_instances_data -- calculate and store L4 forwarding rule metrics - ilb_fwrules.get_forwarding_rules_data -- calculate and store L7 forwarding rule metrics - ilb_fwrules.get_forwarding_rules_data -- calculate and store static routes metrics - routes.get_static_routes_data -- calculate and store peering metrics - peerings.get_vpc_peering_data -- calculate and store peering group metrics - metrics.get_pgg_data - routes.get_routes_ppg -- write buffered timeseries - metrics.flush_series_buffer - - -## Inputs - -direct inputs - -- organization id -- folders (monitored) -- projects (monitored) - -derived inputs - -- projects in folders via CAI -- networks -- subnets -- routers -- peerings -- quotas -- firewall rules -- firewall policies -- routes -- routers -- dynamic routes -- peerings -- instances - -resources - -- project quota -- firewall rules in org via CAI - - key: network -- firewall policies in org via CAI - - key: network -- networks in org via CAI -- subnets in org via CAI - - key: project, network? - - computed metrics: ip usage (used, total, utilization) - - computed metrics: secondary IP ranges -- instances - - key: network - - computed metrics: instance per network (usage, limit, utilization) -- forwarding rules in org via CAI - - key: network, type - - computed metrics: fwd rule per network per type (usage, limit, utilization) -- static routes in org via CAI - - computed metrics: routes per project (usage, limit, utilization) -- dynamic routes via routers - - computed metrics: routes per project (usage, limit, utilization) - - - -## Resources and data - -- projects - - quotas - - -## Metrics - - -## Clients - -- compute -- asset inventory -- momnitoring - From 23c48ba06f245f50258e36bc7fe6b83214ba4ea8 Mon Sep 17 00:00:00 2001 From: Ludo Date: Sun, 20 Nov 2022 17:30:26 +0100 Subject: [PATCH 20/82] streamline --- .../network-dashboard/cf/main.py | 46 ++++++++++++------- .../network-dashboard/cf/plugins/__init__.py | 12 +++-- .../cf/plugins/discover-cai-compute.py | 35 +++++++------- .../cf/plugins/discover-cai-projects.py | 38 +++++++-------- .../cf/plugins/discover-compute-quota.py | 36 +++++++++------ .../plugins/discover-compute-routerstatus.py | 33 +++++++------ .../cf/plugins/discover-metrics.py | 36 +++++++-------- .../{series-networks.py => series-subnets.py} | 0 .../network-dashboard/cf/plugins/utils.py | 2 +- 9 files changed, 133 insertions(+), 105 deletions(-) rename blueprints/cloud-operations/network-dashboard/cf/plugins/{series-networks.py => series-subnets.py} (100%) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 5cdf1e5b9f..a551628ff9 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -23,7 +23,10 @@ from google.auth.transport.requests import AuthorizedSession -HTTP = AuthorizedSession(google.auth.default()[0]) +try: + HTTP = AuthorizedSession(google.auth.default()[0]) +except google.auth.exceptions.RefreshError as e: + raise SystemExit(e.args[0]) LOGGER = logging.getLogger('net-dash') Q_COLLECTION = collections.deque() RESOURCES = {} @@ -32,18 +35,30 @@ def do_discovery(): - LOGGER.info('discovery start') + LOGGER.info(f'discovery start') for plugin in plugins.get_discovery_plugins(): - requests = collections.deque(plugin.func(RESOURCES)) - LOGGER.info(f'discovery {plugin.name} ({len(requests)})') - while requests: - request = requests.popleft() - response = fetch(request) - for next_request in plugin.handler(RESOURCES, response): - if not next_request: + q = collections.deque(plugin.func(RESOURCES)) + while q: + result = q.popleft() + if isinstance(result, plugins.HTTPRequest): + response = fetch(result) + if not response: continue - LOGGER.info(f'discovery {plugin.name} (+1)') - requests.append(next_request) + if result.json: + try: + results = plugin.func(RESOURCES, response, response.json()) + except json.decoder.JSONDecodeError as e: + LOGGER.critical( + f'error decoding JSON for {result.url}: {e.args[0]}') + continue + else: + results = plugin.func(RESOURCES, response) + q += collections.deque(results) + elif isinstance(result, plugins.Resource): + if result.key: + RESOURCES[result.type][result.id][result.key] = result.data + else: + RESOURCES[result.type][result.id] = result.data def do_init(organization, folder, project, op_project): @@ -67,9 +82,9 @@ def fetch(request): response = HTTP.post(request.url, headers=request.headers, data=request.data) if response.status_code != 200: - # TODO: handle this LOGGER.critical( f'response code {response.status_code} for URL {request.url}') + return return response @@ -78,17 +93,14 @@ def fetch(request): help='GCP organization id') @click.option('--op-project', '-op', required=True, type=str, help='GCP monitoring project where metrics will be stored') -@click.option('--project', '-p', required=False, type=str, multiple=True, +@click.option('--project', '-p', type=str, multiple=True, help='GCP project id, can be specified multiple times') -@click.option('--folder', '-p', required=False, type=int, multiple=True, +@click.option('--folder', '-p', type=int, multiple=True, help='GCP folder id, can be specified multiple times') def main(organization=None, op_project=None, project=None, folder=None): logging.basicConfig(level=logging.INFO) - do_init(organization, folder, project, op_project) - do_discovery() - LOGGER.info( {k: len(v) for k, v in RESOURCES.items() if not isinstance(v, str)}) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 002602f640..cdbc4cab23 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -29,11 +29,13 @@ _PLUGINS_INIT = [] _PLUGINS_SERIES = [] -HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data') +HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data json', + defaults=[True]) Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') -Plugin = collections.namedtuple('Plugin', 'func name level priority handler', +Plugin = collections.namedtuple('Plugin', 'func name level priority', defaults=[None, None, None]) -Resource = collections.namedtuple('Resource', 'id data') +Resource = collections.namedtuple('Resource', 'type id data key', + defaults=[None]) class PluginError(Exception): @@ -64,11 +66,11 @@ def _register(collection, func, *args): collection.append(Plugin(func, name, *args)) -def register_discovery(handler_func, level=Level.PRIMARY, priority=99): +def register_discovery(level=Level.PRIMARY, priority=99): 'Register plugins that discover data.' def outer(func): - _register(_PLUGINS_DISCOVERY, func, level, priority, handler_func) + _register(_PLUGINS_DISCOVERY, func, level, priority) return func return outer diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 267f56f651..91516882a5 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -16,7 +16,7 @@ import logging import urllib.parse -from . import HTTPRequest, Level, register_init, register_discovery +from . import HTTPRequest, Level, Resource, register_init, register_discovery from .utils import parse_cai_results # https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network @@ -39,17 +39,14 @@ NAMES = {v: k for k, v in TYPES.items()} -def _handle_discovery(resources, response): +def _handle_discovery(resources, response, data): 'Process discovery data.' - request = response.request LOGGER.info('discovery handle request') - try: - data = response.json() - except json.decoder.JSONDecodeError as e: - LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') - return {} for result in parse_cai_results(data, 'cai-compute', method='list'): - _handle_resource(resources, result['resource']) + resource = _handle_resource(resources, result['resource']) + if not resource: + continue + yield resource page_token = data.get('nextPageToken') if page_token: LOGGER.info('requesting next page') @@ -77,7 +74,7 @@ def _handle_resource(resources, data): if not extra_attrs: return resource.update(extra_attrs) - resources[resource_name][resource['self_link']] = resource + return Resource(resource_name, resource['self_link'], resource) def _handle_addresses(resource, data): @@ -180,9 +177,9 @@ def _get_parent(parent, resources): 'Extract and return resource parent.' parent_type, parent_id = parent.split('/')[-2:] if parent_type == 'projects': - project_id = resources['projects:number'].get(parent_id) - if project_id: - return {'project_id': project_id, 'project_number': parent_id} + project = resources['projects:number'].get(parent_id) + if project: + return {'project_id': project['project_id'], 'project_number': parent_id} if parent_type == 'folders': if int(parent_id) in resources['folders']: return {'parent': f'{parent_type}/{parent_id}'} @@ -208,8 +205,12 @@ def init(resources): resources[name] = {} -@register_discovery(_handle_discovery, Level.PRIMARY, 10) -def start_discovery(resources): +@register_discovery(Level.PRIMARY, 10) +def start_discovery(resources, response=None, data=None): 'Start discovery by returning the asset list URL for asset types.' - LOGGER.info('discovery start') - yield HTTPRequest(_url(resources), {}, None) + LOGGER.info(f'discovery (has response: {response is not None})') + if response is None: + yield HTTPRequest(_url(resources), {}, None) + else: + for result in _handle_discovery(resources, response, data): + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index 8daeb047d4..73cff6b733 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -15,7 +15,7 @@ import json import logging -from . import HTTPRequest, Level, register_init, register_discovery +from . import HTTPRequest, Level, Resource, register_init, register_discovery from .utils import parse_page_token, parse_cai_results LOGGER = logging.getLogger('net-dash.discovery.cai-projects') @@ -28,20 +28,16 @@ f'?assetTypes=cloudresourcemanager.googleapis.com%2FProject&pageSize=500') -def _handle_discovery(resources, response): +def _handle_discovery(resources, response, data): LOGGER.info('discovery handle request') - request = response.request - try: - data = response.json() - except json.decoder.JSONDecodeError as e: - LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') - return {} for result in parse_cai_results(data, NAME, TYPE): - number = result['project'].split('/')[1] - project_id = result['displayName'] - resources[NAME][project_id] = {'number': number} - resources['projects:number'][number] = project_id - next_url = parse_page_token(data, request.url) + data = { + 'number': result['project'].split('/')[1], + 'project_id': result['displayName'] + } + yield Resource('projects', data['project_id'], data) + yield Resource('projects:number', data['number'], data) + next_url = parse_page_token(data, response.request.url) if next_url: LOGGER.info('discovery next url') yield HTTPRequest(next_url, {}, None) @@ -56,9 +52,13 @@ def init(resources): resources['projects:number'] = {} -@register_discovery(_handle_discovery, Level.CORE, 0) -def start_discovery(resources): - LOGGER.info('discovery start') - for resource_type in (NAME, 'folders'): - for k in resources.get(resource_type, []): - yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) +@register_discovery(Level.CORE, 0) +def start_discovery(resources, response=None, data=None): + LOGGER.info(f'discovery (has response: {response is not None})') + if response is None: + for resource_type in (NAME, 'folders'): + for k in resources.get(resource_type, []): + yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) + else: + for result in _handle_discovery(resources, response, data): + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 127e034ac5..4c64c5c730 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -14,7 +14,7 @@ import logging -from . import Level, register_init, register_discovery +from . import Level, Resource, register_init, register_discovery from .utils import batched, dirty_mp_request, dirty_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-quota') @@ -29,7 +29,12 @@ def _handle_discovery(resources, response): content_type = response.headers['content-type'] for part in dirty_mp_response(content_type, response.content): kind = part.get('kind') - quota = part.get('quotas') + quota = { + q['metric']: { + 'limit': q['limit'], + 'usage': q['usage'] + } for q in part.get('quotas', []) + } self_link = part.get('selfLink') if not self_link: logging.warn('invalid quota response') @@ -40,9 +45,9 @@ def _handle_discovery(resources, response): elif kind == 'compute#region': project_id = self_link[-3] region = self_link[-1] - project_quota = resources[NAME].setdefault(project_id, {}) - project_quota[region] = quota - yield + if project_id not in resources[NAME]: + resources[NAME][project_id] = {} + yield Resource(NAME, project_id, quota, region) @register_init @@ -52,11 +57,16 @@ def init(resources): resources[NAME] = {} -@register_discovery(_handle_discovery, Level.DERIVED, 0) -def start_discovery(resources): - LOGGER.info('discovery start') - urls = [API_GLOBAL_URL.format(p) for p in resources['projects']] - if not urls: - return - for batch in batched(urls, 10): - yield dirty_mp_request(batch) +@register_discovery(Level.DERIVED, 0) +def start_discovery(resources, response=None): + LOGGER.info(f'discovery (has response: {response is not None})') + if response is None: + # TODO: regions + urls = [API_GLOBAL_URL.format(p) for p in resources['projects']] + if not urls: + return + for batch in batched(urls, 10): + yield dirty_mp_request(batch) + else: + for result in _handle_discovery(resources, response): + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index f56fb98d84..44fcfbf42d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -14,7 +14,7 @@ import logging -from . import Level, register_init, register_discovery +from . import Level, Resource, register_init, register_discovery from .utils import batched, dirty_mp_request, dirty_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-routes-dynamic') @@ -48,8 +48,9 @@ def _handle_discovery(resources, response): continue num_learned_routes = sum( int(p.get('numLearnedRoutes', 0)) for p in bgp_peer_status) - resources[NAME][router['network']] = resources[NAME].get( - router['network'], 0) + num_learned_routes + yield Resource( + NAME, router['network'], + resources[NAME].get(router['network'], 0) + num_learned_routes) yield @@ -60,14 +61,18 @@ def init(resources): resources[NAME] = {} -@register_discovery(_handle_discovery, Level.DERIVED) -def start_discovery(resources): - LOGGER.info('discovery start') - urls = [ - API_URL.format(r['project_id'], r['region'], r['name']) - for r in resources['routers'].values() - ] - if not urls: - return - for batch in batched(urls, 10): - yield dirty_mp_request(batch) +@register_discovery(Level.DERIVED) +def start_discovery(resources, response=None): + LOGGER.info(f'discovery (has response: {response is not None})') + if not response: + urls = [ + API_URL.format(r['project_id'], r['region'], r['name']) + for r in resources['routers'].values() + ] + if not urls: + return + for batch in batched(urls, 10): + yield dirty_mp_request(batch) + else: + for result in _handle_discovery(resources, response): + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py index c42d3c3c29..0ce91836ae 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py @@ -16,7 +16,7 @@ import logging import urllib.parse -from . import HTTPRequest, Level, register_init, register_discovery +from . import HTTPRequest, Level, Resource, register_init, register_discovery from .utils import parse_page_token, parse_cai_results LOGGER = logging.getLogger('net-dash.discovery.metrics') @@ -29,21 +29,15 @@ '&pageSize=500') -def _handle_discovery(resources, response): +def _handle_discovery(resources, response, data): LOGGER.info('discovery handle request') - request = response.request - try: - data = response.json() - except json.decoder.JSONDecodeError as e: - LOGGER.critical(f'error decoding URL {request.url}: {e.args[0]}') - return {} descriptors = data.get('metricDescriptors') if not descriptors: LOGGER.info('no descriptors found') return for d in descriptors: - resources['metrics'].append(d['type']) - next_url = parse_page_token(data, request.url) + yield Resource('metrics', d['type'], {}) + next_url = parse_page_token(data, response.request.url) if next_url: LOGGER.info('discovery next url') yield HTTPRequest(next_url, {}, None) @@ -53,12 +47,16 @@ def _handle_discovery(resources, response): def init(resources): LOGGER.info('init') if 'metrics' not in resources: - resources['metrics'] = [] - - -@register_discovery(_handle_discovery, Level.CORE, 0) -def start_discovery(resources): - LOGGER.info('discovery start') - yield HTTPRequest( - URL.format(urllib.parse.quote_plus(resources['monitoring_project'])), {}, - None) + resources['metrics'] = {} + + +@register_discovery(Level.CORE, 0) +def start_discovery(resources, response=None, data=None): + LOGGER.info(f'discovery (has response: {response is not None})') + if response is None: + yield HTTPRequest( + URL.format(urllib.parse.quote_plus(resources['monitoring_project'])), + {}, None) + else: + for result in _handle_discovery(resources, response, data): + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index ad891a41c2..eccce24b14 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -55,7 +55,7 @@ def dirty_mp_request(urls, boundary='1234567890'): data.append('--\n') headers = {'content-type': f'multipart/mixed; boundary={boundary[2:]}'} return HTTPRequest('https://compute.googleapis.com/batch/compute/v1', headers, - ''.join(data)) + ''.join(data), False) def dirty_mp_response(content_type, content): From 5e82dd4725d3f8120eaa5de8a57423d6969d91f5 Mon Sep 17 00:00:00 2001 From: Ludo Date: Sun, 20 Nov 2022 19:03:03 +0100 Subject: [PATCH 21/82] fixes, dump --- .../network-dashboard/cf/main.py | 10 +- .../network-dashboard/cf/out.json | 4018 +++++++++++++++++ .../plugins/discover-compute-routerstatus.py | 6 +- .../cf/plugins/discover-metrics.py | 6 +- 4 files changed, 4031 insertions(+), 9 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/out.json diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index a551628ff9..2adbcc9bcf 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -97,15 +97,19 @@ def fetch(request): help='GCP project id, can be specified multiple times') @click.option('--folder', '-p', type=int, multiple=True, help='GCP folder id, can be specified multiple times') -def main(organization=None, op_project=None, project=None, folder=None): +@click.option('--dump', type=click.File('w'), + help='Export JSON representation of resources to file.') +def main(organization=None, op_project=None, project=None, folder=None, + dump=False): logging.basicConfig(level=logging.INFO) do_init(organization, folder, project, op_project) do_discovery() LOGGER.info( {k: len(v) for k, v in RESOURCES.items() if not isinstance(v, str)}) - # import icecream - # icecream.ic(RESOURCES) + if dump: + import json + json.dump(RESOURCES, dump, indent=2) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json new file mode 100644 index 0000000000..8981c1b957 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -0,0 +1,4018 @@ +{ + "organization": "436789450919", + "monitoring_project": "google.com:ludo-sce-test", + "folders": { + "821058723541": {}, + "949988871993": {}, + "321477570496": {} + }, + "projects": { + "tf-playground-simple": { + "number": "64297462517", + "project_id": "tf-playground-simple" + }, + "ludo-dev-net-spoke-0": { + "number": "759592912116", + "project_id": "ludo-dev-net-spoke-0" + }, + "ludo-prod-net-landing-0": { + "number": "233262889141", + "project_id": "ludo-prod-net-landing-0" + }, + "ludo-prod-net-spoke-0": { + "number": "195159130008", + "project_id": "ludo-prod-net-spoke-0" + }, + "ludo-dev-sec-core-0": { + "number": "971935141727", + "project_id": "ludo-dev-sec-core-0" + }, + "ludo-prod-sec-core-0": { + "number": "990827422843", + "project_id": "ludo-prod-sec-core-0" + }, + "tf-playground-gcs-test-0": { + "number": "833078410166", + "project_id": "tf-playground-gcs-test-0" + }, + "tf-playground-svpc-gce-dr": { + "number": "433746214138", + "project_id": "tf-playground-svpc-gce-dr" + }, + "tf-playground-svpc-net-dr": { + "number": "697669426824", + "project_id": "tf-playground-svpc-net-dr" + }, + "tf-playground-svpc-openshift": { + "number": "515444627958", + "project_id": "tf-playground-svpc-openshift" + }, + "tf-playground-svpc-gce": { + "number": "783093469136", + "project_id": "tf-playground-svpc-gce" + }, + "tf-playground-svpc-net": { + "number": "1079408472053", + "project_id": "tf-playground-svpc-net" + }, + "tf-playground-svpc-gke": { + "number": "1043465648801", + "project_id": "tf-playground-svpc-gke" + } + }, + "addresses": { + "projects/ludo-prod-net-landing-0/regions/europe-west1/addresses/dns-forwarding-3bfb2285cc42c149": { + "id": "1086509377706319543", + "name": "dns-forwarding-3bfb2285cc42c149", + "self_link": "projects/ludo-prod-net-landing-0/regions/europe-west1/addresses/dns-forwarding-3bfb2285cc42c149", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "address": "10.128.0.2", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-172afd5c9074083e": { + "id": "7127118736212042670", + "name": "dns-forwarding-172afd5c9074083e", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-172afd5c9074083e", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.0.0.57", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-54cb61094a666746": { + "id": "7222063742818919342", + "name": "dns-forwarding-54cb61094a666746", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-54cb61094a666746", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.0.8.212", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-bd1b9d6abb188fe6": { + "id": "1333839437438854061", + "name": "dns-forwarding-bd1b9d6abb188fe6", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-bd1b9d6abb188fe6", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.0.32.4", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-df6066a8d47814f0": { + "id": "4253770329399436206", + "name": "dns-forwarding-df6066a8d47814f0", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-df6066a8d47814f0", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.0.16.6", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" + }, + "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-79dc52791ec64c76": { + "id": "6033180111970444205", + "name": "dns-forwarding-79dc52791ec64c76", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-79dc52791ec64c76", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.8.0.3", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net" + }, + "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-daf734e63a98deae": { + "id": "569155647513502637", + "name": "dns-forwarding-daf734e63a98deae", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-daf734e63a98deae", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.8.32.4", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce" + }, + "projects/tf-playground-svpc-net/regions/europe-west4/addresses/dns-forwarding-967ac4c986c1c5c3": { + "id": "5851821467174642606", + "name": "dns-forwarding-967ac4c986c1c5c3", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west4/addresses/dns-forwarding-967ac4c986c1c5c3", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.16.32.3", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/addresses/bastion-wg": { + "id": "9106472427605818326", + "name": "bastion-wg", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/bastion-wg", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "34.154.198.58", + "purpose": null, + "internal": false, + "network": null, + "subnet": null + }, + "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b295f9019fda5f74": { + "id": "42185983700394876", + "name": "dns-forwarding-b295f9019fda5f74", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b295f9019fda5f74", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.24.32.2", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b317a059abd2811d": { + "id": "1218744583122739717", + "name": "dns-forwarding-b317a059abd2811d", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b317a059abd2811d", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.255.2.2", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-e1617ac9a49c0ec7": { + "id": "4510536153047654869", + "name": "dns-forwarding-e1617ac9a49c0ec7", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-e1617ac9a49c0ec7", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.24.0.2", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-auto-ip-8742544-2-1664893602541039": { + "id": "4146356766404377677", + "name": "nat-auto-ip-8742544-2-1664893602541039", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-auto-ip-8742544-2-1664893602541039", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "34.154.124.197", + "purpose": "NAT_AUTO", + "internal": false, + "network": null, + "subnet": null + }, + "projects/tf-playground-svpc-net/regions/us-central1/addresses/dns-forwarding-8aa1f3dca28ea24f": { + "id": "2504084894438347729", + "name": "dns-forwarding-8aa1f3dca28ea24f", + "self_link": "projects/tf-playground-svpc-net/regions/us-central1/addresses/dns-forwarding-8aa1f3dca28ea24f", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.0.9.2", + "purpose": "DNS_RESOLVER", + "internal": true, + "network": null, + "subnet": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke" + } + }, + "firewall_policies": { + "locations/global/firewallPolicies/104375135909": { + "id": "104375135909", + "name": "104375135909", + "self_link": "locations/global/firewallPolicies/104375135909", + "parent": "folders/821058723541", + "num_rules": 8, + "num_tuples": 24 + } + }, + "firewalls": { + "projects/ludo-prod-net-landing-0/global/firewalls/allow-onprem-probes-example": { + "id": "7090352733815889570", + "name": "allow-onprem-probes-example", + "self_link": "projects/ludo-prod-net-landing-0/global/firewalls/allow-onprem-probes-example", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-admins": { + "id": "3420407108972372213", + "name": "shared-vpc-ingress-admins", + "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-admins", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-http": { + "id": "2424070100408918260", + "name": "shared-vpc-ingress-tag-http", + "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-http", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-https": { + "id": "7645864452390445302", + "name": "shared-vpc-ingress-tag-https", + "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-https", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-ssh": { + "id": "6993639888575062262", + "name": "shared-vpc-ingress-tag-ssh", + "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-ssh", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-all": { + "id": "644754961722399058", + "name": "gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-all", + "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-all", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-master": { + "id": "4060019673106897235", + "name": "gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-master", + "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-master", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-vms": { + "id": "1839785476750465362", + "name": "gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-vms", + "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-vms", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-composer-nodes": { + "id": "717568057687567763", + "name": "ingress-allow-composer-nodes", + "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-composer-nodes", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-dataflow-load": { + "id": "1577493211258187155", + "name": "ingress-allow-dataflow-load", + "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-dataflow-load", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/tf-playground-svpc-net/global/firewalls/ilb-l7": { + "id": "3828320867690421959", + "name": "ilb-l7", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/ilb-l7", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/ingress-dns": { + "id": "3282453664652777643", + "name": "ingress-dns", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/ingress-dns", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/ingress-rdp": { + "id": "5733710631021458602", + "name": "ingress-rdp", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/ingress-rdp", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/mikro": { + "id": "7384237133743586341", + "name": "mikro", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/mikro", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/proxy": { + "id": "2052097723271132136", + "name": "proxy", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/proxy", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-admins": { + "id": "6595227897868606454", + "name": "shared-vpc-ingress-admins", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-admins", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-http": { + "id": "565554973513112567", + "name": "shared-vpc-ingress-tag-http", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-http", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-https": { + "id": "1628950298606459895", + "name": "shared-vpc-ingress-tag-https", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-https", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-ssh": { + "id": "365815681166629877", + "name": "shared-vpc-ingress-tag-ssh", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-ssh", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + } + }, + "forwarding_rules": { + "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/bastion-wg": { + "id": "4724174290265929790", + "name": "bastion-wg", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/bastion-wg", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "34.154.198.58", + "load_balancing_scheme": "EXTERNAL", + "network": null, + "region": "europe-west8", + "subnet": null + }, + "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/ilb-test": { + "id": "697628543187352862", + "name": "ilb-test", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/ilb-test", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "10.24.32.29", + "load_balancing_scheme": "INTERNAL_MANAGED", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "europe-west8", + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + } + }, + "instances": { + "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/nginx-ew8-b": { + "id": "1193081157111798056", + "name": "nginx-ew8-b", + "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/nginx-ew8-b", + "project_id": "tf-playground-svpc-gce", + "project_number": "783093469136", + "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", + "networks": [ + { + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + } + ] + }, + "projects/tf-playground-svpc-gce/zones/europe-west8-c/instances/nginx-ew8-c": { + "id": "4082958655368747304", + "name": "nginx-ew8-c", + "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-c/instances/nginx-ew8-c", + "project_id": "tf-playground-svpc-gce", + "project_number": "783093469136", + "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-c", + "networks": [ + { + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + } + ] + }, + "projects/tf-playground-svpc-net/zones/europe-west8-b/instances/bastion": { + "id": "837293213121504385", + "name": "bastion", + "self_link": "projects/tf-playground-svpc-net/zones/europe-west8-b/instances/bastion", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/zones/europe-west8-b", + "networks": [ + { + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" + } + ] + } + }, + "networks": { + "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0": { + "id": "2315184905594658556", + "name": "prod-spoke-0", + "self_link": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "peerings": [ + { + "name": "prod-peering-0-prod-spoke-0-prod-landing-0", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + { + "name": "servicenetworking-googleapis-com", + "network": "projects/r63cc730008bdfb49p-tp/global/networks/servicenetworking" + }, + { + "name": "to-dev-spoke-0", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + } + ], + "subnets": [ + "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4", + "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1", + "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4", + "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1" + ] + }, + "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0": { + "id": "2001134848211738357", + "name": "prod-landing-0", + "self_link": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "peerings": [ + { + "name": "dev-peering-0-prod-landing-0-dev-spoke-0", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + { + "name": "prod-peering-0-prod-landing-0-prod-spoke-0", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + } + ], + "subnets": [ + "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" + ] + }, + "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc": { + "id": "1887910606791828228", + "name": "shared-vpc", + "self_link": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "peerings": [], + "subnets": [ + "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb", + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net", + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip", + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce", + "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke", + "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce", + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke", + "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce", + "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce", + "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net", + "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net" + ] + }, + "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0": { + "id": "5618184304347635448", + "name": "dev-spoke-0", + "self_link": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "peerings": [ + { + "name": "dev-peering-0-dev-spoke-0-prod-landing-0", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + { + "name": "servicenetworking-googleapis-com", + "network": "projects/p10d8c97c9bd39799p-tp/global/networks/servicenetworking" + }, + { + "name": "to-prod-spoke-0", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + } + ], + "subnets": [ + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1", + "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8", + "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4", + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1", + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1", + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1" + ] + }, + "projects/tf-playground-svpc-net/global/networks/shared-vpc": { + "id": "6490252338739305443", + "name": "shared-vpc", + "self_link": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "peerings": [ + { + "name": "servicenetworking-googleapis-com", + "network": "projects/a2fed18bfde5785bdp-tp/global/networks/servicenetworking" + } + ], + "subnets": [ + "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce", + "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke", + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net", + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce", + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net", + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke", + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb", + "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net", + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce", + "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce", + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" + ] + } + }, + "subnetworks": { + "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1": { + "id": "4645967677499694788", + "name": "prod-default-ew1", + "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "cidr_range": "10.128.64.0/24", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west1" + }, + "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1": { + "id": "5412553412046524467", + "name": "prod-l7ilb-europe-west1", + "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "cidr_range": "10.128.92.0/24", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "purpose": "REGIONAL_MANAGED_PROXY", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west1" + }, + "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4": { + "id": "6980100425062399320", + "name": "prod-default-ew4", + "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "cidr_range": "10.128.65.0/24", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west4" + }, + "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4": { + "id": "6293569359006437427", + "name": "prod-l7ilb-europe-west4", + "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "cidr_range": "10.128.93.0/24", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "purpose": "REGIONAL_MANAGED_PROXY", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west4" + }, + "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1": { + "id": "6183073250819497646", + "name": "landing-default-ew1", + "self_link": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "cidr_range": "10.128.0.0/24", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-landing-0/regions/europe-west1" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce": { + "id": "1753894793794347243", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.0.32.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke": { + "id": "4511601755917967596", + "name": "gke", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.0.8.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip": { + "id": "5701824999950967019", + "name": "gke-vip", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.0.16.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net": { + "id": "2784995877536981227", + "name": "net", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.0.0.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce": { + "id": "4654054937679159533", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.8.32.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west3" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net": { + "id": "1999120736153034998", + "name": "net", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.8.0.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west3" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce": { + "id": "4593147538303148267", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.16.32.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west4" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce": { + "id": "7339386710113181931", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.24.32.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west8" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb": { + "id": "137392004069122283", + "name": "l7ilb", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.255.2.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "REGIONAL_MANAGED_PROXY", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west8" + }, + "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net": { + "id": "3309179009467998443", + "name": "net", + "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.24.0.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west8" + }, + "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke": { + "id": "748427805746061548", + "name": "gke", + "self_link": "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "cidr_range": "10.0.9.0/24", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/us-central1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1": { + "id": "6460040680782777531", + "name": "dev-dataplatform-ew1", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "cidr_range": "10.128.48.0/24", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1": { + "id": "2831390316161982168", + "name": "dev-default-ew1", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "cidr_range": "10.128.32.0/24", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1": { + "id": "249944920601617510", + "name": "dev-gke-nodes-ew1", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "cidr_range": "10.64.0.0/24", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1": { + "id": "2280726911218651187", + "name": "dev-l7ilb-europe-west1", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "cidr_range": "10.128.60.0/24", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "purpose": "REGIONAL_MANAGED_PROXY", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4": { + "id": "6599648524023664689", + "name": "dev-l7ilb-europe-west4", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "cidr_range": "10.128.61.0/24", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "purpose": "REGIONAL_MANAGED_PROXY", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west4" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8": { + "id": "4456740759944235703", + "name": "ludo-dev-default-ew8", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "cidr_range": "10.128.33.0/24", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west8" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce": { + "id": "1129946310421719129", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.0.32.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke": { + "id": "2478860073774111734", + "name": "gke", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.0.8.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip": { + "id": "1416395609824617176", + "name": "gke-vip", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.0.16.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net": { + "id": "8424841432903474166", + "name": "net", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.0.0.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + }, + "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce": { + "id": "8930552426943163299", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.8.32.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west3" + }, + "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net": { + "id": "5182660640425322036", + "name": "net", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.8.0.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west3" + }, + "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce": { + "id": "2464381379294734425", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.16.32.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west4" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce": { + "id": "6421800903324885909", + "name": "gce", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.24.32.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb": { + "id": "9159849323393344035", + "name": "l7ilb", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.255.2.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "REGIONAL_MANAGED_PROXY", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net": { + "id": "4872843329097064947", + "name": "net", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.24.0.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + }, + "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke": { + "id": "440151099218547687", + "name": "gke", + "self_link": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "10.0.9.0/24", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/us-central1" + } + }, + "routers": { + "projects/ludo-prod-net-spoke-0/regions/europe-west1/routers/prod-nat-ew1-nat": { + "id": "983301949238599311", + "name": "prod-nat-ew1-nat", + "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west1/routers/prod-nat-ew1-nat", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "region": "europe-west1" + }, + "projects/ludo-prod-net-landing-0/regions/europe-west1/routers/prod-nat-ew1": { + "id": "478658513322696364", + "name": "prod-nat-ew1", + "self_link": "projects/ludo-prod-net-landing-0/regions/europe-west1/routers/prod-nat-ew1", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", + "region": "europe-west1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west1/routers/dev-nat-ew1-nat": { + "id": "2223705014177685131", + "name": "dev-nat-ew1-nat", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/routers/dev-nat-ew1-nat", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "region": "europe-west1" + }, + "projects/ludo-dev-net-spoke-0/regions/europe-west8/routers/dev-nat-ew8-nat": { + "id": "5947823903261526678", + "name": "dev-nat-ew8-nat", + "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west8/routers/dev-nat-ew8-nat", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "region": "europe-west8" + }, + "projects/tf-playground-svpc-net/regions/europe-west1/routers/vpc-shared-ew1-nat": { + "id": "3113194381827127169", + "name": "vpc-shared-ew1-nat", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/routers/vpc-shared-ew1-nat", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "europe-west1" + }, + "projects/tf-playground-svpc-net/regions/europe-west3/routers/vpc-shared-ew3-nat": { + "id": "8873531324389668767", + "name": "vpc-shared-ew3-nat", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/routers/vpc-shared-ew3-nat", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "europe-west3" + }, + "projects/tf-playground-svpc-net/regions/europe-west4/routers/vpc-shared-ew4-nat": { + "id": "6860218593541109633", + "name": "vpc-shared-ew4-nat", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west4/routers/vpc-shared-ew4-nat", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "europe-west4" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpc-shared-ew8-nat": { + "id": "7968307069143758750", + "name": "vpc-shared-ew8-nat", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpc-shared-ew8-nat", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "europe-west8" + }, + "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpn-home": { + "id": "3925724850449128568", + "name": "vpn-home", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpn-home", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "europe-west8" + }, + "projects/tf-playground-svpc-net/regions/us-central1/routers/vpc-shared-uc1-nat": { + "id": "441772775809234847", + "name": "vpc-shared-uc1-nat", + "self_link": "projects/tf-playground-svpc-net/regions/us-central1/routers/vpc-shared-uc1-nat", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "region": "us-central1" + } + }, + "routes": { + "projects/ludo-prod-net-spoke-0/global/routes/default-route-40f9b3cd9946750d": { + "id": "899011966177939471", + "name": "default-route-40f9b3cd9946750d", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-40f9b3cd9946750d", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "network", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/default-route-72ef5fa46984f071": { + "id": "8568498928831487711", + "name": "default-route-72ef5fa46984f071", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-72ef5fa46984f071", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "network", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/default-route-ac3bcc52856e80e3": { + "id": "1784782880021188656", + "name": "default-route-ac3bcc52856e80e3", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-ac3bcc52856e80e3", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "network", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/default-route-c51a2cb0324f01a9": { + "id": "2824182523471707477", + "name": "default-route-c51a2cb0324f01a9", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-c51a2cb0324f01a9", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "network", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/default-route-f1fd5dec0cdd76f3": { + "id": "308922196058758903", + "name": "default-route-f1fd5dec0cdd76f3", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-f1fd5dec0cdd76f3", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "gateway", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-0321109d8d96dc9e": { + "id": "3324799109720031011", + "name": "peering-route-0321109d8d96dc9e", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-0321109d8d96dc9e", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-1644420b9cb80ef8": { + "id": "6241664760832264842", + "name": "peering-route-1644420b9cb80ef8", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-1644420b9cb80ef8", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-25f1911afe289c3c": { + "id": "6827131325114597155", + "name": "peering-route-25f1911afe289c3c", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-25f1911afe289c3c", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-578ffc0543e989eb": { + "id": "44456750104929401", + "name": "peering-route-578ffc0543e989eb", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-578ffc0543e989eb", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-780e3930f3b7f24c": { + "id": "5746981242429989896", + "name": "peering-route-780e3930f3b7f24c", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-780e3930f3b7f24c", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e997f107eeecfb2f": { + "id": "8585255017838395171", + "name": "peering-route-e997f107eeecfb2f", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e997f107eeecfb2f", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e9d68888bc28ea31": { + "id": "1139893304295800611", + "name": "peering-route-e9d68888bc28ea31", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e9d68888bc28ea31", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-private-googleapis": { + "id": "2948233411096010438", + "name": "prod-spoke-0-private-googleapis", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-private-googleapis", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "gateway", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-restricted-googleapis": { + "id": "6714004201574293210", + "name": "prod-spoke-0-restricted-googleapis", + "self_link": "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-restricted-googleapis", + "project_id": "ludo-prod-net-spoke-0", + "project_number": "195159130008", + "next_hop_type": "gateway", + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/default-route-ad0e530b51126577": { + "id": "6241659907566788261", + "name": "default-route-ad0e530b51126577", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/default-route-ad0e530b51126577", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "network", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/default-route-b9c9da9f17272c9c": { + "id": "7146043362834964209", + "name": "default-route-b9c9da9f17272c9c", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/default-route-b9c9da9f17272c9c", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "gateway", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-13de4d911be51ea7": { + "id": "3094272727316390945", + "name": "peering-route-13de4d911be51ea7", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-13de4d911be51ea7", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-5401900b4cf2582d": { + "id": "407360634550707234", + "name": "peering-route-5401900b4cf2582d", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-5401900b4cf2582d", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-5914b2470e809033": { + "id": "89240550193980449", + "name": "peering-route-5914b2470e809033", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-5914b2470e809033", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-60914e501bc4499b": { + "id": "5203468670312866824", + "name": "peering-route-60914e501bc4499b", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-60914e501bc4499b", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-78f9f9fb7e76b2c3": { + "id": "4743569086995194914", + "name": "peering-route-78f9f9fb7e76b2c3", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-78f9f9fb7e76b2c3", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-7b4a6991ec5526c2": { + "id": "9040733271652000776", + "name": "peering-route-7b4a6991ec5526c2", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-7b4a6991ec5526c2", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-8c33217d76de1f1b": { + "id": "8099720905059427336", + "name": "peering-route-8c33217d76de1f1b", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-8c33217d76de1f1b", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-93ff9a0739edb2cb": { + "id": "5236433849979724808", + "name": "peering-route-93ff9a0739edb2cb", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-93ff9a0739edb2cb", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-ed15dbfa7467b3cf": { + "id": "3577867237184201761", + "name": "peering-route-ed15dbfa7467b3cf", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-ed15dbfa7467b3cf", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/peering-route-ef500c56c5474741": { + "id": "1283494745588327458", + "name": "peering-route-ef500c56c5474741", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-ef500c56c5474741", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "peering", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-private-googleapis": { + "id": "1170308089481584291", + "name": "prod-landing-0-private-googleapis", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-private-googleapis", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "gateway", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-restricted-googleapis": { + "id": "6281371948279717551", + "name": "prod-landing-0-restricted-googleapis", + "self_link": "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-restricted-googleapis", + "project_id": "ludo-prod-net-landing-0", + "project_number": "233262889141", + "next_hop_type": "gateway", + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-11fa55a9adcb4b70": { + "id": "4585803715457638631", + "name": "default-route-11fa55a9adcb4b70", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-11fa55a9adcb4b70", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-14ce7cda556cb5eb": { + "id": "5125715850850263272", + "name": "default-route-14ce7cda556cb5eb", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-14ce7cda556cb5eb", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-2c2bb1c74a41b25d": { + "id": "3285216510750286050", + "name": "default-route-2c2bb1c74a41b25d", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-2c2bb1c74a41b25d", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-4bbf918c1bea1feb": { + "id": "2274013185856299239", + "name": "default-route-4bbf918c1bea1feb", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-4bbf918c1bea1feb", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-72dd143284332b85": { + "id": "4671000443382718694", + "name": "default-route-72dd143284332b85", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-72dd143284332b85", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-769a73de008d7193": { + "id": "7621429696248733926", + "name": "default-route-769a73de008d7193", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-769a73de008d7193", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-8c42e476440a1bcb": { + "id": "7399270554358046953", + "name": "default-route-8c42e476440a1bcb", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-8c42e476440a1bcb", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-936ac84acce6cd86": { + "id": "2775823425120559336", + "name": "default-route-936ac84acce6cd86", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-936ac84acce6cd86", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-a14eb190bc5b6e46": { + "id": "2616984457793822950", + "name": "default-route-a14eb190bc5b6e46", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-a14eb190bc5b6e46", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-a805f525462ef124": { + "id": "3493013866118701287", + "name": "default-route-a805f525462ef124", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-a805f525462ef124", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-afaa2cc263a7bfec": { + "id": "5888856166968386334", + "name": "default-route-afaa2cc263a7bfec", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-afaa2cc263a7bfec", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "gateway", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-b3acb1437d77cd80": { + "id": "5120529042185183462", + "name": "default-route-b3acb1437d77cd80", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-b3acb1437d77cd80", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-d8ec4a849124c7a0": { + "id": "1115146042057482473", + "name": "default-route-d8ec4a849124c7a0", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-d8ec4a849124c7a0", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-d9d6128120ab85fc": { + "id": "6526128012596142311", + "name": "default-route-d9d6128120ab85fc", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-d9d6128120ab85fc", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-deb1eff327532f1f": { + "id": "7152080901412020456", + "name": "default-route-deb1eff327532f1f", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-deb1eff327532f1f", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-eaa61c921aba25b0": { + "id": "3430978466596839679", + "name": "default-route-eaa61c921aba25b0", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-eaa61c921aba25b0", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/default-route-eccab6ddfa3b0515": { + "id": "93366519110885618", + "name": "default-route-eccab6ddfa3b0515", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-eccab6ddfa3b0515", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net-dr/global/routes/shared-vpc-private-googleapis": { + "id": "4227585992519114998", + "name": "shared-vpc-private-googleapis", + "self_link": "projects/tf-playground-svpc-net-dr/global/routes/shared-vpc-private-googleapis", + "project_id": "tf-playground-svpc-net-dr", + "project_number": "697669426824", + "next_hop_type": "gateway", + "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-1e66f6999d63bbdb": { + "id": "2794396208018312351", + "name": "default-route-1e66f6999d63bbdb", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-1e66f6999d63bbdb", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "network", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-25f5b67fa9b5322e": { + "id": "3153137358666260143", + "name": "default-route-25f5b67fa9b5322e", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-25f5b67fa9b5322e", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "network", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-4a1b1a9528dc9066": { + "id": "662362691274237026", + "name": "default-route-4a1b1a9528dc9066", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-4a1b1a9528dc9066", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "network", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-62198dccd5e11e45": { + "id": "2384952332828478130", + "name": "default-route-62198dccd5e11e45", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-62198dccd5e11e45", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "network", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-bbd81d101d7db2bc": { + "id": "4717561835935272974", + "name": "default-route-bbd81d101d7db2bc", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-bbd81d101d7db2bc", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "network", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-c4da502c04154f5a": { + "id": "7479509588678598387", + "name": "default-route-c4da502c04154f5a", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-c4da502c04154f5a", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "gateway", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/default-route-f42e36a70e595e65": { + "id": "1909548583298568205", + "name": "default-route-f42e36a70e595e65", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-f42e36a70e595e65", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "network", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-private-googleapis": { + "id": "1142085726235582125", + "name": "dev-spoke-0-private-googleapis", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-private-googleapis", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "gateway", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-restricted-googleapis": { + "id": "994791119899385561", + "name": "dev-spoke-0-restricted-googleapis", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-restricted-googleapis", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "gateway", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/peering-route-3d010c7ff6100be5": { + "id": "3538065964205980974", + "name": "peering-route-3d010c7ff6100be5", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-3d010c7ff6100be5", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "peering", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/peering-route-4272e533f3961e2c": { + "id": "2081363471452454691", + "name": "peering-route-4272e533f3961e2c", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-4272e533f3961e2c", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "peering", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/peering-route-47726de7b73f0892": { + "id": "2243680524795077666", + "name": "peering-route-47726de7b73f0892", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-47726de7b73f0892", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "peering", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/peering-route-90e8310f8ac976df": { + "id": "1926683561476603683", + "name": "peering-route-90e8310f8ac976df", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-90e8310f8ac976df", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "peering", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/ludo-dev-net-spoke-0/global/routes/peering-route-9da568659960707b": { + "id": "3178237314998845219", + "name": "peering-route-9da568659960707b", + "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-9da568659960707b", + "project_id": "ludo-dev-net-spoke-0", + "project_number": "759592912116", + "next_hop_type": "peering", + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + "projects/tf-playground-svpc-net/global/routes/default": { + "id": "7660236194664304033", + "name": "default", + "self_link": "projects/tf-playground-svpc-net/global/routes/default", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "gateway", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-00276cedf07156e1": { + "id": "5814005392503551500", + "name": "default-route-00276cedf07156e1", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-00276cedf07156e1", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-1640724c624a81a9": { + "id": "7924010442940425742", + "name": "default-route-1640724c624a81a9", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-1640724c624a81a9", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-1a63661f5467170b": { + "id": "7771813397673697776", + "name": "default-route-1a63661f5467170b", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-1a63661f5467170b", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-3968f730f137e758": { + "id": "5204039853703324562", + "name": "default-route-3968f730f137e758", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-3968f730f137e758", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-406dce5c3cd518ae": { + "id": "477834944705155438", + "name": "default-route-406dce5c3cd518ae", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-406dce5c3cd518ae", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-45df00aec802e6e1": { + "id": "858206022063643636", + "name": "default-route-45df00aec802e6e1", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-45df00aec802e6e1", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-46edb716dee26f2d": { + "id": "1194245977988092270", + "name": "default-route-46edb716dee26f2d", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-46edb716dee26f2d", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-62849020e7572a5a": { + "id": "41053546903447638", + "name": "default-route-62849020e7572a5a", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-62849020e7572a5a", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-6d41e89bfb9eb2f6": { + "id": "1628999936569849356", + "name": "default-route-6d41e89bfb9eb2f6", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-6d41e89bfb9eb2f6", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-6e55995b3ecdce42": { + "id": "2529095030201726477", + "name": "default-route-6e55995b3ecdce42", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-6e55995b3ecdce42", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-7055cdd9fb57cccd": { + "id": "345479256826663886", + "name": "default-route-7055cdd9fb57cccd", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-7055cdd9fb57cccd", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-939636f9c9827425": { + "id": "6330256629342462679", + "name": "default-route-939636f9c9827425", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-939636f9c9827425", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-adfa72880afd062f": { + "id": "3946321220088521248", + "name": "default-route-adfa72880afd062f", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-adfa72880afd062f", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-bd832fca2633024d": { + "id": "3512752011138509755", + "name": "default-route-bd832fca2633024d", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-bd832fca2633024d", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-e618f80d606cfa58": { + "id": "4078275391946775500", + "name": "default-route-e618f80d606cfa58", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-e618f80d606cfa58", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/default-route-fa56ade9f222917e": { + "id": "2528926229423401046", + "name": "default-route-fa56ade9f222917e", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-fa56ade9f222917e", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/iap-ig": { + "id": "3009389033799183115", + "name": "iap-ig", + "self_link": "projects/tf-playground-svpc-net/global/routes/iap-ig", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "gateway", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/peering-route-5686ff94bce71953": { + "id": "4281803293141324207", + "name": "peering-route-5686ff94bce71953", + "self_link": "projects/tf-playground-svpc-net/global/routes/peering-route-5686ff94bce71953", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "peering", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/routes/shared-vpc-private-googleapis": { + "id": "8851580889041732346", + "name": "shared-vpc-private-googleapis", + "self_link": "projects/tf-playground-svpc-net/global/routes/shared-vpc-private-googleapis", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "gateway", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + } + }, + "projects:number": { + "64297462517": { + "number": "64297462517", + "project_id": "tf-playground-simple" + }, + "759592912116": { + "number": "759592912116", + "project_id": "ludo-dev-net-spoke-0" + }, + "233262889141": { + "number": "233262889141", + "project_id": "ludo-prod-net-landing-0" + }, + "195159130008": { + "number": "195159130008", + "project_id": "ludo-prod-net-spoke-0" + }, + "971935141727": { + "number": "971935141727", + "project_id": "ludo-dev-sec-core-0" + }, + "990827422843": { + "number": "990827422843", + "project_id": "ludo-prod-sec-core-0" + }, + "833078410166": { + "number": "833078410166", + "project_id": "tf-playground-gcs-test-0" + }, + "433746214138": { + "number": "433746214138", + "project_id": "tf-playground-svpc-gce-dr" + }, + "697669426824": { + "number": "697669426824", + "project_id": "tf-playground-svpc-net-dr" + }, + "515444627958": { + "number": "515444627958", + "project_id": "tf-playground-svpc-openshift" + }, + "783093469136": { + "number": "783093469136", + "project_id": "tf-playground-svpc-gce" + }, + "1079408472053": { + "number": "1079408472053", + "project_id": "tf-playground-svpc-net" + }, + "1043465648801": { + "number": "1043465648801", + "project_id": "tf-playground-svpc-gke" + } + }, + "quota": { + "tf-playground-simple": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 1 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 2 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 1 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "ludo-dev-net-spoke-0": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 1 + }, + "FIREWALLS": { + "limit": 200, + "usage": 5 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 3 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 2 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 4 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 1 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "ludo-prod-net-landing-0": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 1 + }, + "FIREWALLS": { + "limit": 200, + "usage": 1 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 3 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 1 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "ludo-prod-net-spoke-0": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 1 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 3 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 1 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 1 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "ludo-dev-sec-core-0": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "ludo-prod-sec-core-0": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-gcs-test-0": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-svpc-gce-dr": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-svpc-net-dr": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 1 + }, + "FIREWALLS": { + "limit": 200, + "usage": 4 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 2 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 2 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-svpc-openshift": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-svpc-gce": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 1 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 1 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 9, + "usage": 1 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-svpc-net": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 1 + }, + "FIREWALLS": { + "limit": 200, + "usage": 9 + }, + "IMAGES": { + "limit": 2000, + "usage": 1 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 3 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 1 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 1 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 1 + }, + "URL_MAPS": { + "limit": 30, + "usage": 1 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 1 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 1 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 6 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 2 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 2 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 3 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 1 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 1 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + }, + "tf-playground-svpc-gke": { + "global": { + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "NETWORKS": { + "limit": 15, + "usage": 0 + }, + "FIREWALLS": { + "limit": 200, + "usage": 0 + }, + "IMAGES": { + "limit": 2000, + "usage": 0 + }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "ROUTES": { + "limit": 250, + "usage": 0 + }, + "FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "TARGET_POOLS": { + "limit": 150, + "usage": 0 + }, + "HEALTH_CHECKS": { + "limit": 150, + "usage": 0 + }, + "IN_USE_ADDRESSES": { + "limit": 69, + "usage": 0 + }, + "TARGET_INSTANCES": { + "limit": 150, + "usage": 0 + }, + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "URL_MAPS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "INSTANCE_TEMPLATES": { + "limit": 300, + "usage": 0 + }, + "TARGET_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 + }, + "ROUTERS": { + "limit": 10, + "usage": 0 + }, + "TARGET_SSL_PROXIES": { + "limit": 30, + "usage": 0 + }, + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, + "usage": 0 + }, + "SUBNETWORKS": { + "limit": 175, + "usage": 0 + }, + "TARGET_TCP_PROXIES": { + "limit": 30, + "usage": 0 + }, + "SECURITY_POLICIES": { + "limit": 10, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 0 + }, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, + "usage": 0 + }, + "INTERCONNECTS": { + "limit": 6, + "usage": 0 + }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 0 + }, + "VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 0 + }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, + "usage": 0 + }, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + }, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, + "usage": 0 + } + } + } + }, + "routes-dynamic": { + "projects/tf-playground-svpc-net/global/networks/shared-vpc": { + "vpn-home": 1 + } + }, + "metrics": {}, + "peering-groups": {} +} \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index 44fcfbf42d..3951949341 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -48,9 +48,9 @@ def _handle_discovery(resources, response): continue num_learned_routes = sum( int(p.get('numLearnedRoutes', 0)) for p in bgp_peer_status) - yield Resource( - NAME, router['network'], - resources[NAME].get(router['network'], 0) + num_learned_routes) + if router['network'] not in resources[NAME]: + resources[NAME][router['network']] = {} + yield Resource(NAME, router['network'], num_learned_routes, router['name']) yield diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py index 0ce91836ae..d1d3c6dcc4 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py @@ -36,7 +36,7 @@ def _handle_discovery(resources, response, data): LOGGER.info('no descriptors found') return for d in descriptors: - yield Resource('metrics', d['type'], {}) + yield Resource(NAME, d['type'], {}) next_url = parse_page_token(data, response.request.url) if next_url: LOGGER.info('discovery next url') @@ -46,8 +46,8 @@ def _handle_discovery(resources, response, data): @register_init def init(resources): LOGGER.info('init') - if 'metrics' not in resources: - resources['metrics'] = {} + if NAME not in resources: + resources[NAME] = {} @register_discovery(Level.CORE, 0) From 260fbe5dd9da202335ff869b108b985333b18199 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 08:05:25 +0100 Subject: [PATCH 22/82] streamline --- .../network-dashboard/cf/main.py | 13 +++- .../network-dashboard/cf/out.json | 51 +++++++------- .../network-dashboard/cf/plugins/__init__.py | 66 ++++--------------- .../cf/plugins/discover-cai-compute.py | 8 +-- .../cf/plugins/discover-metrics.py | 3 +- .../cf/plugins/series-subnets.py | 16 +++++ 6 files changed, 71 insertions(+), 86 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 2adbcc9bcf..2d6ee44b06 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -28,8 +28,8 @@ except google.auth.exceptions.RefreshError as e: raise SystemExit(e.args[0]) LOGGER = logging.getLogger('net-dash') -Q_COLLECTION = collections.deque() RESOURCES = {} +TIMESERIES = [] Result = collections.namedtuple('Result', 'phase resource data') @@ -62,17 +62,26 @@ def do_discovery(): def do_init(organization, folder, project, op_project): + LOGGER.info(f'init start') RESOURCES['organization'] = str(organization) RESOURCES['monitoring_project'] = op_project if folder: RESOURCES['folders'] = {f: {} for f in folder} if project: RESOURCES['projects'] = {p: {} for p in project} - for plugin in plugins.get_init_plugins(): plugin.func(RESOURCES) +def do_timeseries(): + LOGGER.info(f'timeseries start') + for plugin in plugins.get_timeseries_plugins(): + for result in plugin.func(RESOURCES): + if not result: + continue + TIMESERIES.append(result) + + def fetch(request): # try LOGGER.info(f'fetch {request.url}') diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 8981c1b957..df59663dcb 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -71,7 +71,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" + "subnetwork": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" }, "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-172afd5c9074083e": { "id": "7127118736212042670", @@ -83,7 +83,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net" }, "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-54cb61094a666746": { "id": "7222063742818919342", @@ -95,7 +95,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke" }, "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-bd1b9d6abb188fe6": { "id": "1333839437438854061", @@ -107,7 +107,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce" }, "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-df6066a8d47814f0": { "id": "4253770329399436206", @@ -119,7 +119,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" }, "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-79dc52791ec64c76": { "id": "6033180111970444205", @@ -131,7 +131,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net" }, "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-daf734e63a98deae": { "id": "569155647513502637", @@ -143,7 +143,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce" }, "projects/tf-playground-svpc-net/regions/europe-west4/addresses/dns-forwarding-967ac4c986c1c5c3": { "id": "5851821467174642606", @@ -155,7 +155,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce" }, "projects/tf-playground-svpc-net/regions/europe-west8/addresses/bastion-wg": { "id": "9106472427605818326", @@ -167,7 +167,7 @@ "purpose": null, "internal": false, "network": null, - "subnet": null + "subnetwork": null }, "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b295f9019fda5f74": { "id": "42185983700394876", @@ -179,7 +179,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" }, "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b317a059abd2811d": { "id": "1218744583122739717", @@ -191,7 +191,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb" }, "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-e1617ac9a49c0ec7": { "id": "4510536153047654869", @@ -203,7 +203,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" }, "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-auto-ip-8742544-2-1664893602541039": { "id": "4146356766404377677", @@ -215,7 +215,7 @@ "purpose": "NAT_AUTO", "internal": false, "network": null, - "subnet": null + "subnetwork": null }, "projects/tf-playground-svpc-net/regions/us-central1/addresses/dns-forwarding-8aa1f3dca28ea24f": { "id": "2504084894438347729", @@ -227,7 +227,7 @@ "purpose": "DNS_RESOLVER", "internal": true, "network": null, - "subnet": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke" + "subnetwork": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke" } }, "firewall_policies": { @@ -405,7 +405,7 @@ "load_balancing_scheme": "EXTERNAL", "network": null, "region": "europe-west8", - "subnet": null + "subnetwork": null }, "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/ilb-test": { "id": "697628543187352862", @@ -417,7 +417,7 @@ "load_balancing_scheme": "INTERNAL_MANAGED", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "region": "europe-west8", - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" } }, "instances": { @@ -431,7 +431,7 @@ "networks": [ { "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" } ] }, @@ -445,7 +445,7 @@ "networks": [ { "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" } ] }, @@ -459,7 +459,7 @@ "networks": [ { "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnet": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" } ] } @@ -485,7 +485,7 @@ "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" } ], - "subnets": [ + "subnetworks": [ "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4", "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1", "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4", @@ -508,7 +508,7 @@ "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" } ], - "subnets": [ + "subnetworks": [ "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" ] }, @@ -519,7 +519,7 @@ "project_id": "tf-playground-svpc-net-dr", "project_number": "697669426824", "peerings": [], - "subnets": [ + "subnetworks": [ "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb", "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net", "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip", @@ -553,7 +553,7 @@ "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" } ], - "subnets": [ + "subnetworks": [ "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1", "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8", "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4", @@ -574,7 +574,7 @@ "network": "projects/a2fed18bfde5785bdp-tp/global/networks/servicenetworking" } ], - "subnets": [ + "subnetworks": [ "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce", "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke", "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net", @@ -4013,6 +4013,5 @@ "vpn-home": 1 } }, - "metrics": {}, - "peering-groups": {} + "metrics": {} } \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index cdbc4cab23..cc4b77131d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -14,6 +14,7 @@ import collections import enum +import functools import importlib import pathlib import pkgutil @@ -24,83 +25,44 @@ 'get_init_plugins', 'register_discovery', 'register_init' ] -_PLUGINS_SERIES = [] _PLUGINS_DISCOVERY = [] _PLUGINS_INIT = [] -_PLUGINS_SERIES = [] +_PLUGINS_TIMESERIES = [] HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data json', defaults=[True]) Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') Plugin = collections.namedtuple('Plugin', 'func name level priority', - defaults=[None, None, None]) + defaults=[Level.PRIMARY, 99]) Resource = collections.namedtuple('Resource', 'type id data key', defaults=[None]) +TimeSeries = collections.namedtuple('TimeSeries', 'metric value labels') class PluginError(Exception): pass -def get_discovery_plugins(): - 'Return discovery plugins.' - for p in _PLUGINS_DISCOVERY: - yield p - - -def get_init_plugins(): - 'Return init plugins.' - for p in _PLUGINS_INIT: - yield p - - -def get_series_plugins(): - 'Return metrics plugins.' - for p in _PLUGINS_SERIES: - yield p - - -def _register(collection, func, *args): +def _register_plugin(collection, *args): 'Derive plugin name from function and add to its collection.' - name = f'{func.__module__}.{func.__name__}' - collection.append(Plugin(func, name, *args)) - - -def register_discovery(level=Level.PRIMARY, priority=99): - 'Register plugins that discover data.' - - def outer(func): - _register(_PLUGINS_DISCOVERY, func, level, priority) - return func - - return outer - - -def register_init(*args): - 'Register plugins that prepare the shared data structure.' if args and type(args[0]) == types.FunctionType: - _register(_PLUGINS_INIT, args[0]) + collection.append( + Plugin(args[0], f'{args[0].__module__}.{args[0].__name__}')) return def outer(func): - _register(_PLUGINS_INIT, func) + collection.append(Plugin(func, f'{func.__module__}.{func.__name__}', *args)) return func return outer -def register_series(*args): - 'Register plugins that derive metrics series from data.' - if args and type(args[0]) == types.FunctionType: - _register(_PLUGINS_SERIES, args[0]) - return - - def outer(func): - _register(_PLUGINS_SERIES, func) - return func - - return outer - +get_discovery_plugins = lambda: iter(_PLUGINS_DISCOVERY) +get_init_plugins = lambda: iter(_PLUGINS_INIT) +get_timeseries_plugins = lambda: iter(_PLUGINS_TIMESERIES) +register_discovery = functools.partial(_register_plugin, _PLUGINS_DISCOVERY) +register_init = functools.partial(_register_plugin, _PLUGINS_INIT) +register_timeseries = functools.partial(_register_plugin, _PLUGINS_TIMESERIES) _plugins_path = str(pathlib.Path(__file__).parent) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 91516882a5..6d8e3d1208 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -86,7 +86,7 @@ def _handle_addresses(resource, data): 'purpose': data.get('purpose'), 'internal': data.get('addressType') == 'INTERNAL', 'network': None if not network else _self_link(network), - 'subnet': None if not subnet else _self_link(subnet) + 'subnetwork': None if not subnet else _self_link(subnet) } @@ -113,7 +113,7 @@ def _handle_forwarding_rules(resource, data): 'load_balancing_scheme': data['loadBalancingScheme'], 'network': None if not network else _self_link(network), 'region': None if not region else region.split('/')[-1], - 'subnet': None if not subnet else _self_link(subnet) + 'subnetwork': None if not subnet else _self_link(subnet) } @@ -123,7 +123,7 @@ def _handle_instances(resource, data): return networks = [{ 'network': _self_link(i['network']), - 'subnet': _self_link(i['subnetwork']) + 'subnetwork': _self_link(i['subnetwork']) } for i in data.get('networkInterfaces', [])] return {'zone': data['zone'], 'networks': networks} @@ -135,7 +135,7 @@ def _handle_networks(resource, data): 'network': _self_link(p['network']) } for p in data.get('peerings', []) if p['state'] == 'ACTIVE'] subnets = [_self_link(s) for s in data.get('subnetworks', [])] - return {'peerings': peerings, 'subnets': subnets} + return {'peerings': peerings, 'subnetworks': subnets} def _handle_routers(resource, data): diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py index d1d3c6dcc4..a0ebd8732d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import urllib.parse from . import HTTPRequest, Level, Resource, register_init, register_discovery -from .utils import parse_page_token, parse_cai_results +from .utils import parse_page_token LOGGER = logging.getLogger('net-dash.discovery.metrics') NAME = 'metrics' diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 6d6d1266c3..28f75a3efe 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -11,3 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import itertools +import logging + +from . import TimeSeries, register_timeseries + +LOGGER = logging.getLogger('net-dash.timeseries.subnets') + + +@register_timeseries +def subnet_timeseries(resources, series): + series = {k: 0 for k, v in resources['subnets']} + vm_networks = itertools.chain.from_iterable( + i['networks'] for i in resources['instances'].values()) + for v in vm_networks: + series[v['subnetwork']] += 1 \ No newline at end of file From 320ffece7f953ddb39a3e5bff2f0760d1faced78 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 08:17:41 +0100 Subject: [PATCH 23/82] remove globals --- .../network-dashboard/cf/main.py | 56 ++++++++++--------- .../cf/plugins/series-subnets.py | 9 ++- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 2d6ee44b06..83702b4471 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -28,16 +28,14 @@ except google.auth.exceptions.RefreshError as e: raise SystemExit(e.args[0]) LOGGER = logging.getLogger('net-dash') -RESOURCES = {} -TIMESERIES = [] Result = collections.namedtuple('Result', 'phase resource data') -def do_discovery(): +def do_discovery(resources): LOGGER.info(f'discovery start') for plugin in plugins.get_discovery_plugins(): - q = collections.deque(plugin.func(RESOURCES)) + q = collections.deque(plugin.func(resources)) while q: result = q.popleft() if isinstance(result, plugins.HTTPRequest): @@ -46,40 +44,40 @@ def do_discovery(): continue if result.json: try: - results = plugin.func(RESOURCES, response, response.json()) + results = plugin.func(resources, response, response.json()) except json.decoder.JSONDecodeError as e: LOGGER.critical( f'error decoding JSON for {result.url}: {e.args[0]}') continue else: - results = plugin.func(RESOURCES, response) + results = plugin.func(resources, response) q += collections.deque(results) elif isinstance(result, plugins.Resource): if result.key: - RESOURCES[result.type][result.id][result.key] = result.data + resources[result.type][result.id][result.key] = result.data else: - RESOURCES[result.type][result.id] = result.data + resources[result.type][result.id] = result.data -def do_init(organization, folder, project, op_project): +def do_init(resources, organization, folder, project, op_project): LOGGER.info(f'init start') - RESOURCES['organization'] = str(organization) - RESOURCES['monitoring_project'] = op_project + resources['organization'] = str(organization) + resources['monitoring_project'] = op_project if folder: - RESOURCES['folders'] = {f: {} for f in folder} + resources['folders'] = {f: {} for f in folder} if project: - RESOURCES['projects'] = {p: {} for p in project} + resources['projects'] = {p: {} for p in project} for plugin in plugins.get_init_plugins(): - plugin.func(RESOURCES) + plugin.func(resources) -def do_timeseries(): +def do_timeseries(resources, timeseries): LOGGER.info(f'timeseries start') for plugin in plugins.get_timeseries_plugins(): - for result in plugin.func(RESOURCES): + for result in plugin.func(resources): if not result: continue - TIMESERIES.append(result) + timeseries.append(result) def fetch(request): @@ -106,19 +104,27 @@ def fetch(request): help='GCP project id, can be specified multiple times') @click.option('--folder', '-p', type=int, multiple=True, help='GCP folder id, can be specified multiple times') -@click.option('--dump', type=click.File('w'), +@click.option('--dump-file', type=click.File('w'), help='Export JSON representation of resources to file.') +@click.option('--load-file', type=click.File('r'), + help='Load JSON resources from file, skips init and discovery.') def main(organization=None, op_project=None, project=None, folder=None, - dump=False): + dump_file=None, load_file=None): logging.basicConfig(level=logging.INFO) - do_init(organization, folder, project, op_project) - do_discovery() + timeseries = [] + if load_file: + resources = json.load(load_file) + else: + resources = {} + do_init(resources, organization, folder, project, op_project) + do_discovery(resources) + do_timeseries(resources, timeseries) LOGGER.info( - {k: len(v) for k, v in RESOURCES.items() if not isinstance(v, str)}) + {k: len(v) for k, v in resources.items() if not isinstance(v, str)}) + LOGGER.info(f'{len(timeseries)} timeseries') - if dump: - import json - json.dump(RESOURCES, dump, indent=2) + if dump_file: + json.dump(resources, dump_file, indent=2) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 28f75a3efe..9ead2bcf52 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -21,9 +21,12 @@ @register_timeseries -def subnet_timeseries(resources, series): - series = {k: 0 for k, v in resources['subnets']} +def subnet_timeseries(resources): + LOGGER.info('timeseries') + series = {k: 0 for k in resources['subnetworks']} vm_networks = itertools.chain.from_iterable( i['networks'] for i in resources['instances'].values()) for v in vm_networks: - series[v['subnetwork']] += 1 \ No newline at end of file + series[v['subnetwork']] += 1 + return + yield \ No newline at end of file From f5969463761159fcfb39065977993848c1479910 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 12:24:12 +0100 Subject: [PATCH 24/82] wip metrics --- .../network-dashboard/cf/main.py | 14 +- .../network-dashboard/cf/out.json | 283 +++++++++++++----- .../cf/plugins/discover-cai-compute.py | 10 +- .../cf/plugins/discover-cai-projects.py | 7 +- .../cf/plugins/discover-metrics.py | 6 +- .../cf/plugins/series-subnets.py | 52 +++- 6 files changed, 271 insertions(+), 101 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 83702b4471..3256af640e 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -59,14 +59,12 @@ def do_discovery(resources): resources[result.type][result.id] = result.data -def do_init(resources, organization, folder, project, op_project): +def do_init(resources, organization, op_project, folders=None, projects=None): LOGGER.info(f'init start') - resources['organization'] = str(organization) - resources['monitoring_project'] = op_project - if folder: - resources['folders'] = {f: {} for f in folder} - if project: - resources['projects'] = {p: {} for p in project} + resources['config:organization'] = str(organization) + resources['config:monitoring_project'] = op_project + resources['config:folders'] = [str(f) for f in folders or []] + resources['config:projects'] = projects or [] for plugin in plugins.get_init_plugins(): plugin.func(resources) @@ -116,7 +114,7 @@ def main(organization=None, op_project=None, project=None, folder=None, resources = json.load(load_file) else: resources = {} - do_init(resources, organization, folder, project, op_project) + do_init(resources, organization, op_project, folder, project) do_discovery(resources) do_timeseries(resources, timeseries) LOGGER.info( diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index df59663dcb..32f07647b6 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -1,66 +1,28 @@ { - "organization": "436789450919", - "monitoring_project": "google.com:ludo-sce-test", - "folders": { - "821058723541": {}, - "949988871993": {}, - "321477570496": {} - }, - "projects": { - "tf-playground-simple": { - "number": "64297462517", - "project_id": "tf-playground-simple" - }, - "ludo-dev-net-spoke-0": { - "number": "759592912116", - "project_id": "ludo-dev-net-spoke-0" - }, - "ludo-prod-net-landing-0": { - "number": "233262889141", - "project_id": "ludo-prod-net-landing-0" - }, - "ludo-prod-net-spoke-0": { - "number": "195159130008", - "project_id": "ludo-prod-net-spoke-0" - }, - "ludo-dev-sec-core-0": { - "number": "971935141727", - "project_id": "ludo-dev-sec-core-0" - }, - "ludo-prod-sec-core-0": { - "number": "990827422843", - "project_id": "ludo-prod-sec-core-0" - }, - "tf-playground-gcs-test-0": { - "number": "833078410166", - "project_id": "tf-playground-gcs-test-0" - }, - "tf-playground-svpc-gce-dr": { - "number": "433746214138", - "project_id": "tf-playground-svpc-gce-dr" - }, - "tf-playground-svpc-net-dr": { - "number": "697669426824", - "project_id": "tf-playground-svpc-net-dr" - }, - "tf-playground-svpc-openshift": { - "number": "515444627958", - "project_id": "tf-playground-svpc-openshift" - }, - "tf-playground-svpc-gce": { - "number": "783093469136", - "project_id": "tf-playground-svpc-gce" - }, - "tf-playground-svpc-net": { - "number": "1079408472053", - "project_id": "tf-playground-svpc-net" - }, - "tf-playground-svpc-gke": { - "number": "1043465648801", - "project_id": "tf-playground-svpc-gke" - } - }, + "config:organization": "436789450919", + "config:monitoring_project": "google.com:ludo-sce-test", + "config:folders": [ + "821058723541", + "949988871993", + "321477570496" + ], + "config:projects": [ + "tf-playground-simple" + ], "addresses": { + "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello": { + "id": "2569728380045293941", + "name": "psc-home-hello", + "self_link": "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "address": "10.24.33.2", + "internal": true, + "purpose": "GCE_ENDPOINT", + "status": "RESERVED", + "network": null, + "subnetwork": "projects/tf-playground-simple/regions/europe-west8/subnetworks/default" + }, "projects/ludo-prod-net-landing-0/regions/europe-west1/addresses/dns-forwarding-3bfb2285cc42c149": { "id": "1086509377706319543", "name": "dns-forwarding-3bfb2285cc42c149", @@ -68,8 +30,9 @@ "project_id": "ludo-prod-net-landing-0", "project_number": "233262889141", "address": "10.128.0.2", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" }, @@ -80,8 +43,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.0.0.57", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net" }, @@ -92,8 +56,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.0.8.212", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke" }, @@ -104,8 +69,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.0.32.4", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce" }, @@ -116,8 +82,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.0.16.6", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" }, @@ -128,8 +95,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.8.0.3", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net" }, @@ -140,8 +108,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.8.32.4", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce" }, @@ -152,8 +121,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.16.32.3", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce" }, @@ -164,8 +134,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "34.154.198.58", - "purpose": null, "internal": false, + "purpose": "", + "status": "IN_USE", "network": null, "subnetwork": null }, @@ -176,8 +147,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.24.32.2", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" }, @@ -188,8 +160,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.255.2.2", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb" }, @@ -200,8 +173,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.24.0.2", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" }, @@ -212,11 +186,25 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "34.154.124.197", - "purpose": "NAT_AUTO", "internal": false, + "purpose": "NAT_AUTO", + "status": "IN_USE", "network": null, "subnetwork": null }, + "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-ingress-ip-1510-1968901406393301314": { + "id": "9125076950352066911", + "name": "nat-ingress-ip-1510-1968901406393301314", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-ingress-ip-1510-1968901406393301314", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "address": "172.16.255.2", + "internal": true, + "purpose": "PSC_PRODUCER_NAT_IP", + "status": "RESERVED", + "network": null, + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc" + }, "projects/tf-playground-svpc-net/regions/us-central1/addresses/dns-forwarding-8aa1f3dca28ea24f": { "id": "2504084894438347729", "name": "dns-forwarding-8aa1f3dca28ea24f", @@ -224,8 +212,9 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "address": "10.0.9.2", - "purpose": "DNS_RESOLVER", "internal": true, + "purpose": "DNS_RESOLVER", + "status": "RESERVED", "network": null, "subnetwork": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke" } @@ -241,6 +230,14 @@ } }, "firewalls": { + "projects/tf-playground-simple/global/firewalls/default-allow-icmp": { + "id": "9189037418640505268", + "name": "default-allow-icmp", + "self_link": "projects/tf-playground-simple/global/firewalls/default-allow-icmp", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "network": "projects/tf-playground-simple/global/networks/default" + }, "projects/ludo-prod-net-landing-0/global/firewalls/allow-onprem-probes-example": { "id": "7090352733815889570", "name": "allow-onprem-probes-example", @@ -395,6 +392,19 @@ } }, "forwarding_rules": { + "projects/tf-playground-simple/regions/europe-west8/forwardingRules/home-hello": { + "id": "1968901406393301314", + "name": "home-hello", + "self_link": "projects/tf-playground-simple/regions/europe-west8/forwardingRules/home-hello", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "address": "10.24.33.2", + "load_balancing_scheme": "", + "network": "projects/tf-playground-simple/global/networks/default", + "psc_accepted": true, + "region": "europe-west8", + "subnetwork": null + }, "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/bastion-wg": { "id": "4724174290265929790", "name": "bastion-wg", @@ -404,6 +414,7 @@ "address": "34.154.198.58", "load_balancing_scheme": "EXTERNAL", "network": null, + "psc_accepted": false, "region": "europe-west8", "subnetwork": null }, @@ -416,6 +427,7 @@ "address": "10.24.32.29", "load_balancing_scheme": "INTERNAL_MANAGED", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "psc_accepted": false, "region": "europe-west8", "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" } @@ -465,6 +477,17 @@ } }, "networks": { + "projects/tf-playground-simple/global/networks/default": { + "id": "8227259653970917801", + "name": "default", + "self_link": "projects/tf-playground-simple/global/networks/default", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "peerings": [], + "subnetworks": [ + "projects/tf-playground-simple/regions/europe-west8/subnetworks/default" + ] + }, "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0": { "id": "2315184905594658556", "name": "prod-spoke-0", @@ -580,6 +603,7 @@ "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net", "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce", "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net", + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc", "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke", "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb", "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net", @@ -590,6 +614,17 @@ } }, "subnetworks": { + "projects/tf-playground-simple/regions/europe-west8/subnetworks/default": { + "id": "521276777991436721", + "name": "default", + "self_link": "projects/tf-playground-simple/regions/europe-west8/subnetworks/default", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "cidr_range": "10.24.33.0/24", + "network": "projects/tf-playground-simple/global/networks/default", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-simple/regions/europe-west8" + }, "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1": { "id": "4645967677499694788", "name": "prod-default-ew1", @@ -942,6 +977,17 @@ "purpose": "PRIVATE", "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" }, + "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc": { + "id": "1816076316698934074", + "name": "psc", + "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "cidr_range": "172.16.255.0/25", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "purpose": "PRIVATE_SERVICE_CONNECT", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + }, "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke": { "id": "440151099218547687", "name": "gke", @@ -1047,6 +1093,24 @@ } }, "routes": { + "projects/tf-playground-simple/global/routes/default-route-709ad165b8383d89": { + "id": "1159999789989296547", + "name": "default-route-709ad165b8383d89", + "self_link": "projects/tf-playground-simple/global/routes/default-route-709ad165b8383d89", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "next_hop_type": "gateway", + "network": "projects/tf-playground-simple/global/networks/default" + }, + "projects/tf-playground-simple/global/routes/default-route-8706754714306526": { + "id": "1359944480556775819", + "name": "default-route-8706754714306526", + "self_link": "projects/tf-playground-simple/global/routes/default-route-8706754714306526", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "next_hop_type": "network", + "network": "projects/tf-playground-simple/global/networks/default" + }, "projects/ludo-prod-net-spoke-0/global/routes/default-route-40f9b3cd9946750d": { "id": "899011966177939471", "name": "default-route-40f9b3cd9946750d", @@ -1668,6 +1732,15 @@ "next_hop_type": "network", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" }, + "projects/tf-playground-svpc-net/global/routes/default-route-648effc960cb3912": { + "id": "3109902933644737334", + "name": "default-route-648effc960cb3912", + "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-648effc960cb3912", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "next_hop_type": "network", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, "projects/tf-playground-svpc-net/global/routes/default-route-6d41e89bfb9eb2f6": { "id": "1628999936569849356", "name": "default-route-6d41e89bfb9eb2f6", @@ -1768,6 +1841,60 @@ "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" } }, + "projects": { + "tf-playground-simple": { + "number": "64297462517", + "project_id": "tf-playground-simple" + }, + "ludo-dev-net-spoke-0": { + "number": "759592912116", + "project_id": "ludo-dev-net-spoke-0" + }, + "ludo-prod-net-landing-0": { + "number": "233262889141", + "project_id": "ludo-prod-net-landing-0" + }, + "ludo-prod-net-spoke-0": { + "number": "195159130008", + "project_id": "ludo-prod-net-spoke-0" + }, + "ludo-dev-sec-core-0": { + "number": "971935141727", + "project_id": "ludo-dev-sec-core-0" + }, + "ludo-prod-sec-core-0": { + "number": "990827422843", + "project_id": "ludo-prod-sec-core-0" + }, + "tf-playground-gcs-test-0": { + "number": "833078410166", + "project_id": "tf-playground-gcs-test-0" + }, + "tf-playground-svpc-gce-dr": { + "number": "433746214138", + "project_id": "tf-playground-svpc-gce-dr" + }, + "tf-playground-svpc-net-dr": { + "number": "697669426824", + "project_id": "tf-playground-svpc-net-dr" + }, + "tf-playground-svpc-openshift": { + "number": "515444627958", + "project_id": "tf-playground-svpc-openshift" + }, + "tf-playground-svpc-gce": { + "number": "783093469136", + "project_id": "tf-playground-svpc-gce" + }, + "tf-playground-svpc-net": { + "number": "1079408472053", + "project_id": "tf-playground-svpc-net" + }, + "tf-playground-svpc-gke": { + "number": "1043465648801", + "project_id": "tf-playground-svpc-gke" + } + }, "projects:number": { "64297462517": { "number": "64297462517", @@ -1831,11 +1958,11 @@ }, "NETWORKS": { "limit": 15, - "usage": 0 + "usage": 1 }, "FIREWALLS": { "limit": 200, - "usage": 0 + "usage": 1 }, "IMAGES": { "limit": 2000, @@ -1847,7 +1974,7 @@ }, "ROUTES": { "limit": 250, - "usage": 0 + "usage": 1 }, "FORWARDING_RULES": { "limit": 45, diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 6d8e3d1208..db53a340e4 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -83,8 +83,9 @@ def _handle_addresses(resource, data): subnet = data.get('subnetwork') return { 'address': data['address'], - 'purpose': data.get('purpose'), 'internal': data.get('addressType') == 'INTERNAL', + 'purpose': data.get('purpose', ''), + 'status': data.get('status', ''), 'network': None if not network else _self_link(network), 'subnetwork': None if not subnet else _self_link(subnet) } @@ -110,8 +111,9 @@ def _handle_forwarding_rules(resource, data): subnet = data.get('subnetwork') return { 'address': data.get('IPAddress'), - 'load_balancing_scheme': data['loadBalancingScheme'], + 'load_balancing_scheme': data.get('loadBalancingScheme', ''), 'network': None if not network else _self_link(network), + 'psc_accepted': data.get('pscConnectionStatus') == 'ACCEPTED', 'region': None if not region else region.split('/')[-1], 'subnetwork': None if not subnet else _self_link(subnet) } @@ -181,7 +183,7 @@ def _get_parent(parent, resources): if project: return {'project_id': project['project_id'], 'project_number': parent_id} if parent_type == 'folders': - if int(parent_id) in resources['folders']: + if parent_id in resources['config:folders']: return {'parent': f'{parent_type}/{parent_id}'} if resources['organization'] == int(parent_id): return {'parent': f'{parent_type}/{parent_id}'} @@ -189,7 +191,7 @@ def _get_parent(parent, resources): def _url(resources): 'Return discovery URL' - organization = resources['organization'] + organization = resources['config:organization'] asset_types = '&'.join( 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) for t in TYPES.values()) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index 73cff6b733..dcd03a6ce8 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -56,9 +56,10 @@ def init(resources): def start_discovery(resources, response=None, data=None): LOGGER.info(f'discovery (has response: {response is not None})') if response is None: - for resource_type in (NAME, 'folders'): - for k in resources.get(resource_type, []): - yield HTTPRequest(CAI_URL.format(f'{resource_type}/{k}'), {}, None) + for v in resources['config:projects']: + yield HTTPRequest(CAI_URL.format(f'projects/{v}'), {}, None) + for v in resources['config:folders']: + yield HTTPRequest(CAI_URL.format(f'folders/{v}'), {}, None) else: for result in _handle_discovery(resources, response, data): yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py index a0ebd8732d..3ee542792d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py @@ -53,9 +53,9 @@ def init(resources): def start_discovery(resources, response=None, data=None): LOGGER.info(f'discovery (has response: {response is not None})') if response is None: - yield HTTPRequest( - URL.format(urllib.parse.quote_plus(resources['monitoring_project'])), - {}, None) + monitoring_project = resources['config:monitoring_project'] + yield HTTPRequest(URL.format(urllib.parse.quote_plus(monitoring_project)), + {}, None) else: for result in _handle_discovery(resources, response, data): yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 9ead2bcf52..2f6c09d0dd 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections +import ipaddress import itertools import logging @@ -20,13 +22,53 @@ LOGGER = logging.getLogger('net-dash.timeseries.subnets') +def _subnet_addresses(resources): + 'Return partial counts of addresses per subnetwork.' + for v in resources['addresses'].values(): + if v['status'] != 'RESERVED': + continue + if v['purpose'] in ('GCE_ENDPOINT', 'DNS_RESOLVER'): + yield v['subnetwork'], 1 + + +def _subnet_forwarding_rules(resources): + 'Return partial counts of forwarding rules per subnetwork.' + for v in resources['forwarding_rules'].values(): + if v['load_balancing_scheme'].startswith('INTERNAL'): + yield v['subnetwork'], 1 + continue + if v['psc_accepted']: + network = resources['networks'].get(v['network']) + if not network: + LOGGER.warn(f'PSC address for missing network {v["network"]}') + continue + address = ipaddress.ip_address(v['address']) + for subnet_self_link in network['subnetworks']: + subnet = resources['subnetworks'][subnet_self_link] + cidr_range = ipaddress.ip_network(subnet['cidr_range']) + if address in cidr_range: + yield subnet_self_link, 1 + break + continue + + +def _subnet_instances(resources): + 'Return partial counts of instances per subnetwork.' + vm_networks = itertools.chain.from_iterable( + i['networks'] for i in resources['instances'].values()) + return collections.Counter(v['subnetwork'] for v in vm_networks).items() + + @register_timeseries def subnet_timeseries(resources): LOGGER.info('timeseries') series = {k: 0 for k in resources['subnetworks']} - vm_networks = itertools.chain.from_iterable( - i['networks'] for i in resources['instances'].values()) - for v in vm_networks: - series[v['subnetwork']] += 1 + counters = itertools.chain(_subnet_addresses(resources), + _subnet_forwarding_rules(resources), + _subnet_instances(resources)) + for subnet, count in counters: + series[subnet] += count + from icecream import ic + ic(series) return - yield \ No newline at end of file + yield From bf340e24839aee999f15e75d87805cbcd08de64d Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 15:33:23 +0100 Subject: [PATCH 25/82] subnet time series --- .../network-dashboard/cf/NOTES.md | 16 +++----- .../network-dashboard/cf/main.py | 3 ++ .../cf/plugins/series-subnets.py | 38 +++++++++++++------ .../cloud-function/metrics/subnets.py | 3 +- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index 9526ed94ce..554676ad9f 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -31,19 +31,19 @@ - get router status `get_routes_for_network` `get_routes_for_router` -- [ ] get and store subnet metrics +- [x] get and store subnet metrics `subnets.get_subnets` - get subnets `get_all_subnets` - calculate subnet utilization `compute_subnet_utilization` - - get instances + - [x] get instances `compute_subnet_utilization_vms` - - get forwarding rules + - [x] get forwarding rules `compute_subnet_utilization_ilbs` - - get addresses + - [x] get addresses `compute_subnet_utilization_addresses` - - get redis instances + - [ ] get redis instances `compute_subnet_utilization_redis` - store metrics - [ ]calculate and store firewall rule metrics @@ -66,7 +66,6 @@ - [ ] write buffered timeseries `metrics.flush_series_buffer` - ## Inputs direct inputs @@ -114,20 +113,15 @@ resources - dynamic routes via routers - computed metrics: routes per project (usage, limit, utilization) - - ## Resources and data - projects - quotas - ## Metrics - ## Clients - compute - asset inventory - momnitoring - diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 3256af640e..26432994ad 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -124,6 +124,9 @@ def main(organization=None, op_project=None, project=None, folder=None, if dump_file: json.dump(resources, dump_file, indent=2) + from icecream import ic + ic(timeseries) + if __name__ == '__main__': main(auto_envvar_prefix='NETMON') diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 2f6c09d0dd..65e2253392 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -22,6 +22,11 @@ LOGGER = logging.getLogger('net-dash.timeseries.subnets') +def _self_link(s): + 'Add initial part to self links.' + return f'https://www.googleapis.com/compute/v1/{s}' + + def _subnet_addresses(resources): 'Return partial counts of addresses per subnetwork.' for v in resources['addresses'].values(): @@ -31,7 +36,7 @@ def _subnet_addresses(resources): yield v['subnetwork'], 1 -def _subnet_forwarding_rules(resources): +def _subnet_forwarding_rules(resources, subnet_nets): 'Return partial counts of forwarding rules per subnetwork.' for v in resources['forwarding_rules'].values(): if v['load_balancing_scheme'].startswith('INTERNAL'): @@ -44,9 +49,7 @@ def _subnet_forwarding_rules(resources): continue address = ipaddress.ip_address(v['address']) for subnet_self_link in network['subnetworks']: - subnet = resources['subnetworks'][subnet_self_link] - cidr_range = ipaddress.ip_network(subnet['cidr_range']) - if address in cidr_range: + if address in subnet_nets[subnet_self_link]: yield subnet_self_link, 1 break continue @@ -62,13 +65,26 @@ def _subnet_instances(resources): @register_timeseries def subnet_timeseries(resources): LOGGER.info('timeseries') + subnet_nets = { + k: ipaddress.ip_network(v['cidr_range']) + for k, v in resources['subnetworks'].items() + } series = {k: 0 for k in resources['subnetworks']} + # TODO: PSA counters = itertools.chain(_subnet_addresses(resources), - _subnet_forwarding_rules(resources), + _subnet_forwarding_rules(resources, subnet_nets), _subnet_instances(resources)) - for subnet, count in counters: - series[subnet] += count - from icecream import ic - ic(series) - return - yield + for subnet_self_link, count in counters: + series[subnet_self_link] += count + for subnet_self_link, count in series.items(): + subnet = resources['subnetworks'][subnet_self_link] + labels = { + 'network': _self_link(subnet['network']), + 'project': subnet['project_id'], + 'subnetwork': _self_link(subnet['id']) + } + max_ips = subnet_nets[subnet_self_link].num_addresses - 4 + yield TimeSeries('subnets/available_addresses', max_ips, labels) + yield TimeSeries('subnets/used_addresses', count, labels) + yield TimeSeries('subnets/used_addresses_ratio', + 0 if count == 0 else count / max_ips, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py index cb16c9c87d..46fbc7564a 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py @@ -149,7 +149,8 @@ def compute_subnet_utilization_ilbs(config, read_mask, all_subnets_dict): for versioned in asset.versioned_resources: for field_name, field_value in versioned.resource.items(): if 'loadBalancingScheme' in field_name and field_value in [ - 'INTERNAL', 'INTERNAL_MANAGED']: + 'INTERNAL', 'INTERNAL_MANAGED' + ]: internal = True # We want to count only accepted PSC endpoint Forwarding Rule # If the PSC endpoint Forwarding Rule is pending, we will count it in the reserved IP addresses From 37f74def188f337ee7ea9a243d5f134a3408d7d3 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 16:22:48 +0100 Subject: [PATCH 26/82] networks per project plugin --- .../network-dashboard/cf/main.py | 4 +- .../network-dashboard/cf/out.json | 75 ++++++++++++++++++- .../cf/plugins/discover-group-networks.py | 37 +++++++++ .../cf/plugins/series-subnets.py | 8 +- 4 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 26432994ad..f6e0a39a2b 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -124,8 +124,8 @@ def main(organization=None, op_project=None, project=None, folder=None, if dump_file: json.dump(resources, dump_file, indent=2) - from icecream import ic - ic(timeseries) + # from icecream import ic + # ic(timeseries) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 32f07647b6..bada07eacd 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -238,6 +238,14 @@ "project_number": "64297462517", "network": "projects/tf-playground-simple/global/networks/default" }, + "projects/tf-playground-simple/global/firewalls/test-allow-icmp": { + "id": "7916104499034728911", + "name": "test-allow-icmp", + "self_link": "projects/tf-playground-simple/global/firewalls/test-allow-icmp", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "network": "projects/tf-playground-simple/global/networks/test" + }, "projects/ludo-prod-net-landing-0/global/firewalls/allow-onprem-probes-example": { "id": "7090352733815889570", "name": "allow-onprem-probes-example", @@ -488,6 +496,17 @@ "projects/tf-playground-simple/regions/europe-west8/subnetworks/default" ] }, + "projects/tf-playground-simple/global/networks/test": { + "id": "3222287547582786016", + "name": "test", + "self_link": "projects/tf-playground-simple/global/networks/test", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "peerings": [], + "subnetworks": [ + "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default" + ] + }, "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0": { "id": "2315184905594658556", "name": "prod-spoke-0", @@ -625,6 +644,17 @@ "purpose": "PRIVATE", "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-simple/regions/europe-west8" }, + "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default": { + "id": "3192067775436040655", + "name": "test-default", + "self_link": "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "cidr_range": "10.0.0.0/24", + "network": "projects/tf-playground-simple/global/networks/test", + "purpose": "PRIVATE", + "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-simple/regions/europe-west8" + }, "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1": { "id": "4645967677499694788", "name": "prod-default-ew1", @@ -1093,6 +1123,15 @@ } }, "routes": { + "projects/tf-playground-simple/global/routes/default-route-1e14cf2fe4b89f0b": { + "id": "1065219576982546940", + "name": "default-route-1e14cf2fe4b89f0b", + "self_link": "projects/tf-playground-simple/global/routes/default-route-1e14cf2fe4b89f0b", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "next_hop_type": "gateway", + "network": "projects/tf-playground-simple/global/networks/test" + }, "projects/tf-playground-simple/global/routes/default-route-709ad165b8383d89": { "id": "1159999789989296547", "name": "default-route-709ad165b8383d89", @@ -1111,6 +1150,15 @@ "next_hop_type": "network", "network": "projects/tf-playground-simple/global/networks/default" }, + "projects/tf-playground-simple/global/routes/default-route-94ba9f38b7d02917": { + "id": "6942400975983749580", + "name": "default-route-94ba9f38b7d02917", + "self_link": "projects/tf-playground-simple/global/routes/default-route-94ba9f38b7d02917", + "project_id": "tf-playground-simple", + "project_number": "64297462517", + "next_hop_type": "network", + "network": "projects/tf-playground-simple/global/networks/test" + }, "projects/ludo-prod-net-spoke-0/global/routes/default-route-40f9b3cd9946750d": { "id": "899011966177939471", "name": "default-route-40f9b3cd9946750d", @@ -1958,11 +2006,11 @@ }, "NETWORKS": { "limit": 15, - "usage": 1 + "usage": 2 }, "FIREWALLS": { "limit": 200, - "usage": 1 + "usage": 2 }, "IMAGES": { "limit": 2000, @@ -1974,7 +2022,7 @@ }, "ROUTES": { "limit": 250, - "usage": 1 + "usage": 2 }, "FORWARDING_RULES": { "limit": 45, @@ -4140,5 +4188,26 @@ "vpn-home": 1 } }, + "networks:project": { + "tf-playground-simple": [ + "projects/tf-playground-simple/global/networks/default", + "projects/tf-playground-simple/global/networks/test" + ], + "ludo-prod-net-spoke-0": [ + "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + ], + "ludo-prod-net-landing-0": [ + "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + ], + "tf-playground-svpc-net-dr": [ + "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" + ], + "ludo-dev-net-spoke-0": [ + "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + ], + "tf-playground-svpc-net": [ + "projects/tf-playground-svpc-net/global/networks/shared-vpc" + ] + }, "metrics": {} } \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py new file mode 100644 index 0000000000..d2e4b51021 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py @@ -0,0 +1,37 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging + +from . import Level, Resource, register_init, register_discovery + +LOGGER = logging.getLogger('net-dash.discovery.compute-routes-dynamic') +NAME = 'networks:project' + + +@register_init +def init(resources): + LOGGER.info('init') + if NAME not in resources: + resources[NAME] = {} + + +@register_discovery(Level.DERIVED) +def start_discovery(resources, response=None): + LOGGER.info(f'discovery (has response: {response is not None})') + grouped = itertools.groupby(resources['networks'].values(), + lambda v: v['project_id']) + for project_id, vpcs in grouped: + yield Resource(NAME, project_id, [v['self_link'] for v in vpcs]) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 65e2253392..3853776fd4 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -69,21 +69,21 @@ def subnet_timeseries(resources): k: ipaddress.ip_network(v['cidr_range']) for k, v in resources['subnetworks'].items() } - series = {k: 0 for k in resources['subnetworks']} + subnet_counts = {k: 0 for k in resources['subnetworks']} # TODO: PSA counters = itertools.chain(_subnet_addresses(resources), _subnet_forwarding_rules(resources, subnet_nets), _subnet_instances(resources)) for subnet_self_link, count in counters: - series[subnet_self_link] += count - for subnet_self_link, count in series.items(): + subnet_counts[subnet_self_link] += count + for subnet_self_link, count in subnet_counts.items(): + max_ips = subnet_nets[subnet_self_link].num_addresses - 4 subnet = resources['subnetworks'][subnet_self_link] labels = { 'network': _self_link(subnet['network']), 'project': subnet['project_id'], 'subnetwork': _self_link(subnet['id']) } - max_ips = subnet_nets[subnet_self_link].num_addresses - 4 yield TimeSeries('subnets/available_addresses', max_ips, labels) yield TimeSeries('subnets/used_addresses', count, labels) yield TimeSeries('subnets/used_addresses_ratio', From 2d9422a92e2feaab8738722f73f5fe97ad99c34a Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 16:48:06 +0100 Subject: [PATCH 27/82] firewall rules timeseries --- .../network-dashboard/cf/NOTES.md | 60 ------------------- .../network-dashboard/cf/main.py | 4 +- .../network-dashboard/cf/out.json | 2 +- .../cf/plugins/discover-cai-compute.py | 4 +- .../cf/plugins/series-firewall-rules.py | 43 +++++++++++++ .../cf/plugins/series-subnets.py | 6 +- 6 files changed, 51 insertions(+), 68 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index 554676ad9f..fd44942612 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -65,63 +65,3 @@ `routes.get_routes_ppg` - [ ] write buffered timeseries `metrics.flush_series_buffer` - -## Inputs - -direct inputs - -- organization id -- folders (monitored) -- projects (monitored) - -derived inputs - -- projects in folders via CAI -- networks -- subnets -- routers -- peerings -- quotas -- firewall rules -- firewall policies -- routes -- routers -- dynamic routes -- peerings -- instances - -resources - -- project quota -- firewall rules in org via CAI - - key: network -- firewall policies in org via CAI - - key: network -- networks in org via CAI -- subnets in org via CAI - - key: project, network? - - computed metrics: ip usage (used, total, utilization) - - computed metrics: secondary IP ranges -- instances - - key: network - - computed metrics: instance per network (usage, limit, utilization) -- forwarding rules in org via CAI - - key: network, type - - computed metrics: fwd rule per network per type (usage, limit, utilization) -- static routes in org via CAI - - computed metrics: routes per project (usage, limit, utilization) -- dynamic routes via routers - - computed metrics: routes per project (usage, limit, utilization) - -## Resources and data - -- projects - - quotas - -## Metrics - -## Clients - -- compute -- asset inventory -- momnitoring diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index f6e0a39a2b..26432994ad 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -124,8 +124,8 @@ def main(organization=None, op_project=None, project=None, folder=None, if dump_file: json.dump(resources, dump_file, indent=2) - # from icecream import ic - # ic(timeseries) + from icecream import ic + ic(timeseries) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index bada07eacd..76229c101c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -229,7 +229,7 @@ "num_tuples": 24 } }, - "firewalls": { + "firewall_rules": { "projects/tf-playground-simple/global/firewalls/default-allow-icmp": { "id": "9189037418640505268", "name": "default-allow-icmp", diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index db53a340e4..7cd4656a78 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -28,7 +28,7 @@ TYPES = { 'addresses': 'Address', 'firewall_policies': 'FirewallPolicy', - 'firewalls': 'Firewall', + 'firewall_rules': 'Firewall', 'forwarding_rules': 'ForwardingRule', 'instances': 'Instance', 'networks': 'Network', @@ -99,7 +99,7 @@ def _handle_firewall_policies(resource, data): } -def _handle_firewalls(resource, data): +def _handle_firewall_rules(resource, data): 'Handle firewall type resource data.' return {'network': _self_link(data['network'])} diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py new file mode 100644 index 0000000000..e8376dd547 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -0,0 +1,43 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging + +from . import TimeSeries, register_timeseries + +LOGGER = logging.getLogger('net-dash.timeseries.firewall-rules') + + +@register_timeseries +def subnet_timeseries(resources): + LOGGER.info('timeseries') + grouped = itertools.groupby(resources['firewall_rules'].values(), + lambda v: v['network']) + for network_id, rules in grouped: + count = len(list(rules)) + labels = { + 'network_name': resources['networks'][network_id]['name'], + 'project_id': resources['networks'][network_id]['project_id'] + } + yield TimeSeries('network/firewalls/used', count, labels) + grouped = itertools.groupby(resources['firewall_rules'].values(), + lambda v: v['project_id']) + for project_id, rules in grouped: + count = len(list(rules)) + limit = int(resources['quota'][project_id]['global']['FIREWALLS']['limit']) + labels = {'project_id': project_id} + yield TimeSeries('project/firewalls/used', count, labels) + yield TimeSeries('project/firewalls/available', limit, labels) + yield TimeSeries('project/firewalls/ratio', count / limit, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 3853776fd4..a8701c1e9a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -84,7 +84,7 @@ def subnet_timeseries(resources): 'project': subnet['project_id'], 'subnetwork': _self_link(subnet['id']) } - yield TimeSeries('subnets/available_addresses', max_ips, labels) - yield TimeSeries('subnets/used_addresses', count, labels) - yield TimeSeries('subnets/used_addresses_ratio', + yield TimeSeries('subnetworks/addresses/available', max_ips, labels) + yield TimeSeries('subnetworks/addresses/used', count, labels) + yield TimeSeries('subnetworks/addresses/ratio', 0 if count == 0 else count / max_ips, labels) From b5ffc135df16ca1b5aaeda0c82eb7876559481e7 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 17:13:32 +0100 Subject: [PATCH 28/82] use names in metric labels --- .../cloud-operations/network-dashboard/cf/NOTES.md | 2 +- .../network-dashboard/cf/plugins/series-subnets.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index fd44942612..efbf5404db 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -46,7 +46,7 @@ - [ ] get redis instances `compute_subnet_utilization_redis` - store metrics -- [ ]calculate and store firewall rule metrics +- [x]calculate and store firewall rule metrics `vpc_firewalls.get_firewalls_data` - [ ] calculate and store firewall policy metrics `firewall_policies.get_firewal_policies_data` diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index a8701c1e9a..bec0651f82 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -22,11 +22,6 @@ LOGGER = logging.getLogger('net-dash.timeseries.subnets') -def _self_link(s): - 'Add initial part to self links.' - return f'https://www.googleapis.com/compute/v1/{s}' - - def _subnet_addresses(resources): 'Return partial counts of addresses per subnetwork.' for v in resources['addresses'].values(): @@ -80,9 +75,9 @@ def subnet_timeseries(resources): max_ips = subnet_nets[subnet_self_link].num_addresses - 4 subnet = resources['subnetworks'][subnet_self_link] labels = { - 'network': _self_link(subnet['network']), + 'network': resources['networks'][subnet['network']]['name'], 'project': subnet['project_id'], - 'subnetwork': _self_link(subnet['id']) + 'subnetwork': subnet['name'] } yield TimeSeries('subnetworks/addresses/available', max_ips, labels) yield TimeSeries('subnetworks/addresses/used', count, labels) From 42168fdb0e716db1d094e7c03f0131ab50ad8065 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 21 Nov 2022 20:57:31 +0100 Subject: [PATCH 29/82] firewall policies timeseries --- .../cf/plugins/series-firewall-policies.py | 33 +++++++++++++++++++ .../cf/plugins/series-firewall-rules.py | 7 ++-- .../cf/plugins/series-subnets.py | 6 ++-- 3 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py new file mode 100644 index 0000000000..d534e363f5 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging + +from . import TimeSeries, register_timeseries + +LOGGER = logging.getLogger('net-dash.timeseries.firewall-policies') +TUPLE_LIMIT = 2000 + + +@register_timeseries +def subnet_timeseries(resources): + LOGGER.info('timeseries') + for v in resources['firewall_policies'].values(): + tuples = int(v['num_tuples']) + labels = {'parent': v['parent'], 'name': v['name']} + yield TimeSeries('firewall_policy/tuples_used', tuples, labels) + yield TimeSeries('firewall_policy/tuples_available', TUPLE_LIMIT, labels) + yield TimeSeries('firewalls_policy/tuples_used_ratio', tuples / TUPLE_LIMIT, + labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index e8376dd547..e9ccc89c24 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -38,6 +38,7 @@ def subnet_timeseries(resources): count = len(list(rules)) limit = int(resources['quota'][project_id]['global']['FIREWALLS']['limit']) labels = {'project_id': project_id} - yield TimeSeries('project/firewalls/used', count, labels) - yield TimeSeries('project/firewalls/available', limit, labels) - yield TimeSeries('project/firewalls/ratio', count / limit, labels) + yield TimeSeries('project/firewall_rules_used', count, labels) + yield TimeSeries('project/firewalls_rules_available', limit, labels) + yield TimeSeries('project/firewalls_rules_used_ratio', count / limit, + labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index bec0651f82..f363e926c5 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -79,7 +79,7 @@ def subnet_timeseries(resources): 'project': subnet['project_id'], 'subnetwork': subnet['name'] } - yield TimeSeries('subnetworks/addresses/available', max_ips, labels) - yield TimeSeries('subnetworks/addresses/used', count, labels) - yield TimeSeries('subnetworks/addresses/ratio', + yield TimeSeries('subnetworks/addresses_available', max_ips, labels) + yield TimeSeries('subnetworks/addresses_used', count, labels) + yield TimeSeries('subnetworks/addresses_used_ratio', 0 if count == 0 else count / max_ips, labels) From d177ecc0ca31cedcd278b9d4ea757b5681561014 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 22 Nov 2022 07:15:51 +0100 Subject: [PATCH 30/82] wip --- blueprints/cloud-operations/network-dashboard/cf/NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index efbf5404db..5af976d052 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -48,7 +48,7 @@ - store metrics - [x]calculate and store firewall rule metrics `vpc_firewalls.get_firewalls_data` -- [ ] calculate and store firewall policy metrics +- [x] calculate and store firewall policy metrics `firewall_policies.get_firewal_policies_data` - [ ] calculate and store instance per network metrics `instances.get_gce_instances_data` From e6d85712788bb48720c010697a886bb79873e74b Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 22 Nov 2022 07:49:41 +0100 Subject: [PATCH 31/82] instances per network timeseries --- .../cf/plugins/series-firewall-policies.py | 3 +- .../cf/plugins/series-firewall-rules.py | 2 +- .../cf/plugins/series-networks.py | 41 +++++++++++++++++++ .../cf/plugins/series-subnets.py | 2 +- 4 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py index d534e363f5..2d38a22fa8 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools import logging from . import TimeSeries, register_timeseries @@ -22,7 +21,7 @@ @register_timeseries -def subnet_timeseries(resources): +def timeseries(resources): LOGGER.info('timeseries') for v in resources['firewall_policies'].values(): tuples = int(v['num_tuples']) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index e9ccc89c24..caf41aca2c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -21,7 +21,7 @@ @register_timeseries -def subnet_timeseries(resources): +def timeseries(resources): LOGGER.info('timeseries') grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['network']) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py new file mode 100644 index 0000000000..6949662074 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -0,0 +1,41 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import itertools +import logging +import operator + +from . import TimeSeries, register_timeseries + +LIMIT = 15000 +LOGGER = logging.getLogger('net-dash.timeseries.networks') + + +@register_timeseries +def timeseries(resources): + LOGGER.info('timeseries') + instance_networks = functools.reduce( + operator.add, [i['networks'] for i in resources['instances'].values()]) + grouped = itertools.groupby(instance_networks, lambda i: i['network']) + for network_id, elements in grouped: + network = resources['networks'].get(network_id) + if not network: + LOGGER.info(f'out of scope instance network {network_id}') + continue + count = len(list(elements)) + labels = {'project': network['project_id'], 'network': network['name']} + yield TimeSeries('network/instances_used', count, labels) + yield TimeSeries('network/instances_available', LIMIT, labels) + yield TimeSeries('network/instances_used_ratio', count / LIMIT, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index f363e926c5..482bd85f57 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -58,7 +58,7 @@ def _subnet_instances(resources): @register_timeseries -def subnet_timeseries(resources): +def timeseries(resources): LOGGER.info('timeseries') subnet_nets = { k: ipaddress.ip_network(v['cidr_range']) From d0cc32b58daa15c43415e1069d34197dc90b2709 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 22 Nov 2022 10:54:22 +0100 Subject: [PATCH 32/82] routes timeseries --- .../network-dashboard/cf/NOTES.md | 6 +- .../network-dashboard/cf/main.py | 31 ++++--- .../network-dashboard/cf/out.json | 6 +- .../plugins/discover-compute-routerstatus.py | 5 +- .../cf/plugins/series-networks.py | 83 ++++++++++++++++--- .../cf/plugins/series-routes.py | 61 ++++++++++++++ .../cf/plugins/series-subnets.py | 6 +- 7 files changed, 164 insertions(+), 34 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index 5af976d052..32f3ff82a6 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -50,11 +50,11 @@ `vpc_firewalls.get_firewalls_data` - [x] calculate and store firewall policy metrics `firewall_policies.get_firewal_policies_data` -- [ ] calculate and store instance per network metrics +- [x] calculate and store instance per network metrics `instances.get_gce_instances_data` -- [ ] calculate and store L4 forwarding rule metrics +- [x] calculate and store L4 forwarding rule metrics `ilb_fwrules.get_forwarding_rules_data` -- [ ] calculate and store L7 forwarding rule metrics +- [x] calculate and store L7 forwarding rule metrics `ilb_fwrules.get_forwarding_rules_data` - [ ] calculate and store static routes metrics `routes.get_static_routes_data` diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 26432994ad..be06805404 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -23,10 +23,7 @@ from google.auth.transport.requests import AuthorizedSession -try: - HTTP = AuthorizedSession(google.auth.default()[0]) -except google.auth.exceptions.RefreshError as e: - raise SystemExit(e.args[0]) +HTTP = AuthorizedSession(google.auth.default()[0]) LOGGER = logging.getLogger('net-dash') Result = collections.namedtuple('Result', 'phase resource data') @@ -69,9 +66,12 @@ def do_init(resources, organization, op_project, folders=None, projects=None): plugin.func(resources) -def do_timeseries(resources, timeseries): - LOGGER.info(f'timeseries start') +def do_timeseries(resources, timeseries, debug_plugin=None): + LOGGER.info(f'timeseries start (debug plugin: {debug_plugin})') for plugin in plugins.get_timeseries_plugins(): + if debug_plugin and plugin.name != debug_plugin: + LOGGER.info(f'skipping {plugin.name}') + continue for result in plugin.func(resources): if not result: continue @@ -81,11 +81,14 @@ def do_timeseries(resources, timeseries): def fetch(request): # try LOGGER.info(f'fetch {request.url}') - if not request.data: - response = HTTP.get(request.url, headers=request.headers) - else: - response = HTTP.post(request.url, headers=request.headers, - data=request.data) + try: + if not request.data: + response = HTTP.get(request.url, headers=request.headers) + else: + response = HTTP.post(request.url, headers=request.headers, + data=request.data) + except google.auth.exceptions.RefreshError as e: + raise SystemExit(e.args[0]) if response.status_code != 200: LOGGER.critical( f'response code {response.status_code} for URL {request.url}') @@ -106,8 +109,10 @@ def fetch(request): help='Export JSON representation of resources to file.') @click.option('--load-file', type=click.File('r'), help='Load JSON resources from file, skips init and discovery.') +@click.option('--debug-plugin', + help='Run only core and specified timeseries plugin.') def main(organization=None, op_project=None, project=None, folder=None, - dump_file=None, load_file=None): + dump_file=None, load_file=None, debug_plugin=None): logging.basicConfig(level=logging.INFO) timeseries = [] if load_file: @@ -116,7 +121,7 @@ def main(organization=None, op_project=None, project=None, folder=None, resources = {} do_init(resources, organization, op_project, folder, project) do_discovery(resources) - do_timeseries(resources, timeseries) + do_timeseries(resources, timeseries, debug_plugin) LOGGER.info( {k: len(v) for k, v in resources.items() if not isinstance(v, str)}) LOGGER.info(f'{len(timeseries)} timeseries') diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 76229c101c..d41109c9d1 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -19,7 +19,7 @@ "address": "10.24.33.2", "internal": true, "purpose": "GCE_ENDPOINT", - "status": "RESERVED", + "status": "IN_USE", "network": null, "subnetwork": "projects/tf-playground-simple/regions/europe-west8/subnetworks/default" }, @@ -4183,9 +4183,9 @@ } } }, - "routes-dynamic": { + "routes_dynamic": { "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "vpn-home": 1 + "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpn-home": 1 } }, "networks:project": { diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index 3951949341..cd185e004f 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -18,7 +18,7 @@ from .utils import batched, dirty_mp_request, dirty_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-routes-dynamic') -NAME = 'routes-dynamic' +NAME = 'routes_dynamic' API_URL = '/compute/v1/projects/{}/regions/{}/routers/{}/getRouterStatus' @@ -50,7 +50,8 @@ def _handle_discovery(resources, response): int(p.get('numLearnedRoutes', 0)) for p in bgp_peer_status) if router['network'] not in resources[NAME]: resources[NAME][router['network']] = {} - yield Resource(NAME, router['network'], num_learned_routes, router['name']) + yield Resource(NAME, router['network'], num_learned_routes, + router['self_link']) yield diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index 6949662074..df580906d1 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -19,23 +19,86 @@ from . import TimeSeries, register_timeseries -LIMIT = 15000 +LIMITS = { + 'INSTANCES_PER_NETWORK_GLOBAL': 15000, + 'INTERNAL_FORWARDING_RULES_PER_NETWORK': 500, + 'INTERNAL_MANAGED_FORWARDING_RULES_PER_NETWORK': 75, + 'ROUTES': 250, + 'SUBNET_RANGES_PER_NETWORK': 300 +} LOGGER = logging.getLogger('net-dash.timeseries.networks') -@register_timeseries -def timeseries(resources): - LOGGER.info('timeseries') +def _group_timeseries(name, resources, grouped, limit_name): + for network_id, elements in grouped: + network = resources['networks'].get(network_id) + if not network: + LOGGER.info(f'out of scope {name} network {network_id}') + continue + count = len(list(elements)) + labels = {'project': network['project_id'], 'network': network['name']} + quota = resources['quota'][network['project_id']] + limit = quota.get(limit_name, LIMITS[limit_name]) + yield TimeSeries(f'network/{name}_used', count, labels) + yield TimeSeries(f'network/{name}_available', limit, labels) + yield TimeSeries(f'network/{name}_used_ratio', count / limit, labels) + + +def _forwarding_rules(resources): + filter = lambda n, v: v['load_balancing_scheme'] != n + forwarding_rules = resources['forwarding_rules'].values() + forwarding_rules_l4 = itertools.filterfalse( + functools.partial(filter, 'INTERNAL'), forwarding_rules) + forwarding_rules_l7 = itertools.filterfalse( + functools.partial(filter, 'INTERNAL_MANAGED'), forwarding_rules) + grouped_l4 = itertools.groupby(forwarding_rules_l4, lambda i: i['network']) + grouped_l7 = itertools.groupby(forwarding_rules_l7, lambda i: i['network']) + return itertools.chain( + _group_timeseries('forwarding_rule_l4', resources, grouped_l4, + 'INTERNAL_FORWARDING_RULES_PER_NETWORK'), + _group_timeseries('forwarding_rule_l7', resources, grouped_l7, + 'INTERNAL_MANAGED_FORWARDING_RULES_PER_NETWORK'), + ) + + +def _instances(resources): instance_networks = functools.reduce( operator.add, [i['networks'] for i in resources['instances'].values()]) grouped = itertools.groupby(instance_networks, lambda i: i['network']) + return _group_timeseries('instances', resources, grouped, + 'INSTANCES_PER_NETWORK_GLOBAL') + + +def _routes(resources): + filter = lambda v: v['next_hop_type'] in ('peering', 'network') + grouped = itertools.groupby(resources['routes'].values(), + lambda v: v['network']) + project_counts = {} for network_id, elements in grouped: network = resources['networks'].get(network_id) - if not network: - LOGGER.info(f'out of scope instance network {network_id}') - continue count = len(list(elements)) labels = {'project': network['project_id'], 'network': network['name']} - yield TimeSeries('network/instances_used', count, labels) - yield TimeSeries('network/instances_available', LIMIT, labels) - yield TimeSeries('network/instances_used_ratio', count / LIMIT, labels) + yield TimeSeries('network/routes_static_used', count, labels) + project_counts[network['project_id']] = project_counts.get( + network['project_id'], 0) + count + for project_id, count in project_counts.items(): + labels = {'project': project_id} + quota = resources['quota'][project_id] + limit = quota.get('ROUTES', LIMITS['ROUTES']) + yield TimeSeries('project/static_routes_used', count, labels) + yield TimeSeries('project/static_routes_available', limit, labels) + yield TimeSeries('project/static_routes_used_ratio', count / limit, labels) + + +def _subnet_ranges(resources): + grouped = itertools.groupby(resources['subnetworks'].values(), + lambda v: v['network']) + return _group_timeseries('subnet', resources, grouped, + 'SUBNET_RANGES_PER_NETWORK') + + +@register_timeseries +def timeseries(resources): + LOGGER.info('timeseries') + return itertools.chain(_forwarding_rules(resources), _instances(resources), + _routes(resources), _subnet_ranges(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py new file mode 100644 index 0000000000..2209aae0a4 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py @@ -0,0 +1,61 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import itertools +import logging +import operator + +from . import TimeSeries, register_timeseries + +LIMITS = {'ROUTES': 250, 'ROUTES_DYNAMIC': 100} +LOGGER = logging.getLogger('net-dash.timeseries.routes') + + +def _dynamic(resources): + for network_id, router_counts in resources['routes_dynamic'].items(): + network = resources['networks'][network_id] + count = sum(router_counts.values()) + labels = {'project': network['project_id'], 'network': network['name']} + limit = LIMITS['ROUTES_DYNAMIC'] + yield TimeSeries('network/routes_dynamic_used', count, labels) + yield TimeSeries('network/routes_dynamic_available', limit, labels) + yield TimeSeries('network/routes_dynamic_used_ratio', count / limit, labels) + + +def _static(resources): + filter = lambda v: v['next_hop_type'] in ('peering', 'network') + routes = itertools.filterfalse(filter, resources['routes'].values()) + grouped = itertools.groupby(routes, lambda v: v['network']) + project_counts = {} + for network_id, elements in grouped: + network = resources['networks'].get(network_id) + count = len(list(elements)) + labels = {'project': network['project_id'], 'network': network['name']} + yield TimeSeries('network/routes_static_used', count, labels) + project_counts[network['project_id']] = project_counts.get( + network['project_id'], 0) + count + for project_id, count in project_counts.items(): + labels = {'project': project_id} + quota = resources['quota'][project_id] + limit = quota.get('ROUTES', LIMITS['ROUTES']) + yield TimeSeries('project/routes_static_used', count, labels) + yield TimeSeries('project/routes_static_available', limit, labels) + yield TimeSeries('project/routes_static_used_ratio', count / limit, labels) + + +@register_timeseries +def timeseries(resources): + LOGGER.info('timeseries') + return itertools.chain(_static(resources), _dynamic(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 482bd85f57..2697d5a00c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -79,7 +79,7 @@ def timeseries(resources): 'project': subnet['project_id'], 'subnetwork': subnet['name'] } - yield TimeSeries('subnetworks/addresses_available', max_ips, labels) - yield TimeSeries('subnetworks/addresses_used', count, labels) - yield TimeSeries('subnetworks/addresses_used_ratio', + yield TimeSeries('subnetwork/addresses_available', max_ips, labels) + yield TimeSeries('subnetwork/addresses_used', count, labels) + yield TimeSeries('subnetwork/addresses_used_ratio', 0 if count == 0 else count / max_ips, labels) From 061a16e6b1a9bab9afff7f9921e5a11554d91e44 Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 22 Nov 2022 16:54:32 +0100 Subject: [PATCH 33/82] custom quota --- .../network-dashboard/cf/NOTES.md | 7 +- .../network-dashboard/cf/custom-quotas.sample | 7 + .../network-dashboard/cf/main.py | 18 +- .../network-dashboard/cf/out.json | 2090 +++++++++-------- .../cf/plugins/discover-cai-compute.py | 1 - .../cf/plugins/discover-cai-projects.py | 1 - .../cf/plugins/discover-compute-quota.py | 10 +- .../cf/plugins/series-firewall-policies.py | 1 + .../cf/plugins/series-firewall-rules.py | 1 + .../cf/plugins/series-networks.py | 28 +- .../cf/plugins/series-peerings.py | 55 + .../cf/plugins/series-routes.py | 5 +- .../cf/plugins/series-subnets.py | 1 + 13 files changed, 1158 insertions(+), 1067 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index 32f3ff82a6..ab1dd68cfb 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -56,12 +56,15 @@ `ilb_fwrules.get_forwarding_rules_data` - [x] calculate and store L7 forwarding rule metrics `ilb_fwrules.get_forwarding_rules_data` -- [ ] calculate and store static routes metrics +- [x] calculate and store static routes metrics `routes.get_static_routes_data` -- [ ] calculate and store peering metrics +- [x] calculate and store peering metrics `peerings.get_vpc_peering_data` - [ ] calculate and store peering group metrics `metrics.get_pgg_data` `routes.get_routes_ppg` - [ ] write buffered timeseries `metrics.flush_series_buffer` +- [x] add per-network and per-project hidden quota override + - [x] implement a custom quota override mechanism + - [x] use it in timeseries plugins diff --git a/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample b/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample new file mode 100644 index 0000000000..61085d2b30 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample @@ -0,0 +1,7 @@ +projects: + tf-playground-svpc-net: + INTERNAL_FORWARDING_RULES_PER_NETWORK: 750 +networks: + # TODO: what are the quotas that can be overridden at the network level? + projects/tf-playground-svpc-net/global/networks/shared-vpc: + PEERINGS_PER_NETWORK: 40 diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index be06805404..bf81019505 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -20,6 +20,7 @@ import click import google.auth import plugins +import yaml from google.auth.transport.requests import AuthorizedSession @@ -56,12 +57,14 @@ def do_discovery(resources): resources[result.type][result.id] = result.data -def do_init(resources, organization, op_project, folders=None, projects=None): +def do_init(resources, organization, op_project, folders=None, projects=None, + custom_quota=None): LOGGER.info(f'init start') resources['config:organization'] = str(organization) resources['config:monitoring_project'] = op_project resources['config:folders'] = [str(f) for f in folders or []] resources['config:projects'] = projects or [] + resources['config:custom_quota'] = custom_quota or {} for plugin in plugins.get_init_plugins(): plugin.func(resources) @@ -105,6 +108,8 @@ def fetch(request): help='GCP project id, can be specified multiple times') @click.option('--folder', '-p', type=int, multiple=True, help='GCP folder id, can be specified multiple times') +@click.option('--custom-quota-file', type=click.File('r'), + help='Custom quota file in yaml format.') @click.option('--dump-file', type=click.File('w'), help='Export JSON representation of resources to file.') @click.option('--load-file', type=click.File('r'), @@ -112,14 +117,21 @@ def fetch(request): @click.option('--debug-plugin', help='Run only core and specified timeseries plugin.') def main(organization=None, op_project=None, project=None, folder=None, - dump_file=None, load_file=None, debug_plugin=None): + custom_quota_file=None, dump_file=None, load_file=None, + debug_plugin=None): logging.basicConfig(level=logging.INFO) timeseries = [] if load_file: resources = json.load(load_file) else: + custom_quota = {} resources = {} - do_init(resources, organization, op_project, folder, project) + if custom_quota_file: + try: + custom_quota = yaml.load(custom_quota_file, Loader=yaml.Loader) + except yaml.YAMLError as e: + raise SystemExit(f'Error decoding custom quota file: {e.args[0]}') + do_init(resources, organization, op_project, folder, project, custom_quota) do_discovery(resources) do_timeseries(resources, timeseries, debug_plugin) LOGGER.info( diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index d41109c9d1..a266d50ecc 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -9,6 +9,18 @@ "config:projects": [ "tf-playground-simple" ], + "config:custom_quota": { + "projects": { + "tf-playground-svpc-net": { + "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 + } + }, + "networks": { + "projects/tf-playground-svpc-net/global/networks/shared-vpc": { + "PEERINGS_PER_NETWORK": 40 + } + } + }, "addresses": { "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello": { "id": "2569728380045293941", @@ -2000,683 +2012,687 @@ "quota": { "tf-playground-simple": { "global": { - "SNAPSHOTS": { - "limit": 5000, - "usage": 1 + "BACKEND_BUCKETS": { + "limit": 9, + "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, - "usage": 2 + "usage": 0 }, "FIREWALLS": { "limit": 200, "usage": 2 }, - "IMAGES": { - "limit": 2000, - "usage": 2 - }, - "STATIC_ADDRESSES": { - "limit": 21, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "ROUTES": { - "limit": 250, - "usage": 2 - }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 + "IMAGES": { + "limit": 2000, + "usage": 2 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 + "MACHINE_IMAGES": { + "limit": 2000, + "usage": 1 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 + "usage": 2 }, - "ROUTERS": { - "limit": 10, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "TARGET_SSL_PROXIES": { + "NETWORK_FIREWALL_POLICIES": { "limit": 30, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTERS": { + "limit": 10, "usage": 0 }, + "ROUTES": { + "limit": 250, + "usage": 2 + }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 + "SNAPSHOTS": { + "limit": 5000, + "usage": 1 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 1 + "TARGET_HTTP_PROXIES": { + "limit": 30, + "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "ludo-dev-net-spoke-0": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, - "usage": 1 + "usage": 0 }, "FIREWALLS": { "limit": 200, "usage": 5 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, "FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 1 + }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, + "usage": 1 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "VPN_TUNNELS": { + "NETWORK_FIREWALL_POLICIES": { "limit": 30, "usage": 0 }, - "BACKEND_BUCKETS": { - "limit": 9, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, "ROUTERS": { "limit": 10, "usage": 2 }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 + "ROUTES": { + "limit": 250, + "usage": 3 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 4 + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 1 + "SUBNETWORKS": { + "limit": 175, + "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 4 } } }, "ludo-prod-net-landing-0": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, - "usage": 1 + "usage": 0 }, "FIREWALLS": { "limit": 200, "usage": 1 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, + "usage": 1 + }, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "VPN_TUNNELS": { + "NETWORK_FIREWALL_POLICIES": { "limit": 30, "usage": 0 }, - "BACKEND_BUCKETS": { - "limit": 9, + "PACKET_MIRRORINGS": { + "limit": 45, + "usage": 0 + }, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, + "usage": 0 + }, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, "ROUTERS": { "limit": 10, "usage": 1 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, + "usage": 3 + }, + "SECURITY_POLICIES": { + "limit": 10, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, + "SECURITY_POLICY_RULES": { + "limit": 100, + "usage": 0 + }, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, "SSL_CERTIFICATES": { "limit": 30, "usage": 0 }, + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 + }, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, + "usage": 0 + }, "SUBNETWORKS": { "limit": 175, "usage": 0 }, - "TARGET_TCP_PROXIES": { + "TARGET_HTTPS_PROXIES": { "limit": 30, "usage": 0 }, - "SECURITY_POLICIES": { - "limit": 10, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_RULES": { - "limit": 100, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "URL_MAPS": { + "limit": 30, "usage": 0 }, "VPN_GATEWAYS": { "limit": 15, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "VPN_TUNNELS": { + "limit": 30, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + } + } + }, + "ludo-prod-net-spoke-0": { + "global": { + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "BACKEND_SERVICES": { + "limit": 75, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, + "FIREWALLS": { + "limit": 200, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "FORWARDING_RULES": { "limit": 45, "usage": 0 }, "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 - } - } - }, - "ludo-prod-net-spoke-0": { - "global": { - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 }, - "NETWORKS": { - "limit": 15, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 1 }, - "FIREWALLS": { - "limit": 200, + "HEALTH_CHECKS": { + "limit": 150, "usage": 0 }, "IMAGES": { "limit": 2000, "usage": 0 }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, - "FORWARDING_RULES": { - "limit": 45, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "HEALTH_CHECKS": { - "limit": 150, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, "IN_USE_ADDRESSES": { "limit": 69, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "URL_MAPS": { - "limit": 30, - "usage": 0 + "NETWORKS": { + "limit": 15, + "usage": 1 }, - "BACKEND_SERVICES": { - "limit": 75, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "BACKEND_BUCKETS": { - "limit": 9, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, "ROUTERS": { "limit": 10, "usage": 1 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, + "usage": 3 + }, + "SECURITY_POLICIES": { + "limit": 10, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "SECURITY_POLICY_RULES": { + "limit": 100, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "TARGET_TCP_PROXIES": { + "SSL_CERTIFICATES": { "limit": 30, "usage": 0 }, - "SECURITY_POLICIES": { - "limit": 10, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "SECURITY_POLICY_RULES": { - "limit": 100, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 1 - }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { + "TARGET_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "URL_MAPS": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "VPN_TUNNELS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "ludo-dev-sec-core-0": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -2684,167 +2700,167 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "ludo-prod-sec-core-0": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -2852,167 +2868,167 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "tf-playground-gcs-test-0": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -3020,167 +3036,167 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "tf-playground-svpc-gce-dr": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -3188,335 +3204,335 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "tf-playground-svpc-net-dr": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { - "limit": 15, - "usage": 1 + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { + "limit": 15, + "usage": 0 }, "FIREWALLS": { "limit": 200, "usage": 4 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "ROUTES": { - "limit": 250, - "usage": 2 - }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 + "usage": 1 }, - "ROUTERS": { - "limit": 10, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "TARGET_SSL_PROXIES": { + "NETWORK_FIREWALL_POLICIES": { "limit": 30, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTERS": { + "limit": 10, "usage": 0 }, + "ROUTES": { + "limit": 250, + "usage": 2 + }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 2 + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 2 } } }, "tf-playground-svpc-openshift": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -3524,167 +3540,167 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 + "STATIC_ADDRESSES": { + "limit": 21, + "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "tf-playground-svpc-gce": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 9, + "usage": 1 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -3692,167 +3708,167 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 1 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 1 }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 + "IMAGES": { + "limit": 2000, + "usage": 1 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 9, - "usage": 1 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, + "NETWORKS": { + "limit": 15, "usage": 0 }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } }, "tf-playground-svpc-net": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 1 }, @@ -3860,167 +3876,170 @@ "limit": 200, "usage": 9 }, - "IMAGES": { - "limit": 2000, - "usage": 1 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, "FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, + "usage": 3 + }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 1 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 1 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 1 + "INTERCONNECTS": { + "limit": 6, + "usage": 0 }, - "URL_MAPS": { - "limit": 30, - "usage": 1 - }, - "BACKEND_SERVICES": { - "limit": 75, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 1 }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, + "NETWORKS": { + "limit": 15, "usage": 1 }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, - "usage": 6 - }, - "TARGET_SSL_PROXIES": { + "NETWORK_FIREWALL_POLICIES": { "limit": 30, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 2 - }, - "SUBNETWORKS": { - "limit": 175, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, + "ROUTERS": { + "limit": 10, + "usage": 6 + }, + "ROUTES": { + "limit": 250, + "usage": 3 + }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, + "usage": 0 + }, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 2 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 3 + "TARGET_HTTPS_PROXIES": { + "limit": 30, + "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 1 }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 + "TARGET_INSTANCES": { + "limit": 150, + "usage": 1 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 1 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, - "usage": 0 + "usage": 1 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 + "VPN_GATEWAYS": { + "limit": 15, + "usage": 1 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 + "VPN_TUNNELS": { + "limit": 30, + "usage": 1 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, + "usage": 2 + }, + "INTERNAL_FORWARDING_RULES_PER_NETWORK": { + "limit": 750 } } }, "tf-playground-svpc-gke": { "global": { - "SNAPSHOTS": { - "limit": 5000, + "BACKEND_BUCKETS": { + "limit": 9, "usage": 0 }, - "NETWORKS": { + "BACKEND_SERVICES": { + "limit": 75, + "usage": 0 + }, + "EXTERNAL_VPN_GATEWAYS": { "limit": 15, "usage": 0 }, @@ -4028,159 +4047,160 @@ "limit": 200, "usage": 0 }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "ROUTES": { - "limit": 250, + "FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "FORWARDING_RULES": { + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { "limit": 45, "usage": 0 }, - "TARGET_POOLS": { - "limit": 150, + "GLOBAL_INTERNAL_ADDRESSES": { + "limit": 5000, "usage": 0 }, "HEALTH_CHECKS": { "limit": 150, "usage": 0 }, - "IN_USE_ADDRESSES": { - "limit": 69, + "IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_INSTANCES": { - "limit": 150, + "INSTANCE_TEMPLATES": { + "limit": 300, "usage": 0 }, - "TARGET_HTTP_PROXIES": { - "limit": 30, + "INTERCONNECTS": { + "limit": 6, "usage": 0 }, - "URL_MAPS": { - "limit": 30, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { + "limit": 45, "usage": 0 }, - "BACKEND_SERVICES": { - "limit": 75, + "IN_USE_ADDRESSES": { + "limit": 69, "usage": 0 }, - "INSTANCE_TEMPLATES": { - "limit": 300, + "MACHINE_IMAGES": { + "limit": 2000, "usage": 0 }, - "TARGET_VPN_GATEWAYS": { + "NETWORKS": { "limit": 15, "usage": 0 }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "BACKEND_BUCKETS": { - "limit": 9, + "NETWORK_ENDPOINT_GROUPS": { + "limit": 300, "usage": 0 }, - "ROUTERS": { - "limit": 10, + "NETWORK_FIREWALL_POLICIES": { + "limit": 30, "usage": 0 }, - "TARGET_SSL_PROXIES": { - "limit": 30, + "PACKET_MIRRORINGS": { + "limit": 45, "usage": 0 }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, + "PUBLIC_ADVERTISED_PREFIXES": { + "limit": 1, "usage": 0 }, - "SSL_CERTIFICATES": { - "limit": 30, + "PUBLIC_DELEGATED_PREFIXES": { + "limit": 10, "usage": 0 }, - "SUBNETWORKS": { - "limit": 175, + "ROUTERS": { + "limit": 10, "usage": 0 }, - "TARGET_TCP_PROXIES": { - "limit": 30, + "ROUTES": { + "limit": 250, "usage": 0 }, "SECURITY_POLICIES": { "limit": 10, "usage": 0 }, + "SECURITY_POLICY_CEVAL_RULES": { + "limit": 20, + "usage": 0 + }, "SECURITY_POLICY_RULES": { "limit": 100, "usage": 0 }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, + "SNAPSHOTS": { + "limit": 5000, "usage": 0 }, - "PACKET_MIRRORINGS": { - "limit": 45, + "SSL_CERTIFICATES": { + "limit": 30, "usage": 0 }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, + "STATIC_ADDRESSES": { + "limit": 21, "usage": 0 }, - "INTERCONNECTS": { - "limit": 6, + "STATIC_BYOIP_ADDRESSES": { + "limit": 1024, "usage": 0 }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, + "SUBNETWORKS": { + "limit": 175, "usage": 0 }, - "VPN_GATEWAYS": { - "limit": 15, + "TARGET_HTTPS_PROXIES": { + "limit": 30, "usage": 0 }, - "MACHINE_IMAGES": { - "limit": 2000, + "TARGET_HTTP_PROXIES": { + "limit": 30, "usage": 0 }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, + "TARGET_INSTANCES": { + "limit": 150, "usage": 0 }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, + "TARGET_POOLS": { + "limit": 150, "usage": 0 }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, + "TARGET_SSL_PROXIES": { + "limit": 30, "usage": 0 }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, + "TARGET_TCP_PROXIES": { + "limit": 30, "usage": 0 }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, + "TARGET_VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "NETWORK_FIREWALL_POLICIES": { + "URL_MAPS": { "limit": 30, "usage": 0 }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, + "VPN_GATEWAYS": { + "limit": 15, "usage": 0 }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, + "VPN_TUNNELS": { + "limit": 30, + "usage": 0 + }, + "XPN_SERVICE_PROJECTS": { + "limit": 1000, "usage": 0 } } + }, + "projects/tf-playground-svpc-net/global/networks/shared-vpc": { + "PEERINGS_PER_NETWORK": { + "limit": 40 + } } }, "routes_dynamic": { diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 7cd4656a78..b156063143 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import urllib.parse diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index dcd03a6ce8..0730c369d6 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import logging from . import HTTPRequest, Level, Resource, register_init, register_discovery diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 4c64c5c730..48abbf7558 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -27,13 +27,14 @@ def _handle_discovery(resources, response): LOGGER.info('discovery handle request') content_type = response.headers['content-type'] + per_project_quota = resources['config:custom_quota'].get('projects') for part in dirty_mp_response(content_type, response.content): kind = part.get('kind') quota = { q['metric']: { 'limit': q['limit'], 'usage': q['usage'] - } for q in part.get('quotas', []) + } for q in sorted(part.get('quotas', []), key=lambda v: v['metric']) } self_link = part.get('selfLink') if not self_link: @@ -42,6 +43,9 @@ def _handle_discovery(resources, response): if kind == 'compute#project': project_id = self_link[-1] region = 'global' + for k, v in per_project_quota.get(project_id, {}).items(): + metric = quota.setdefault(k, {}) + metric['limit'] = v elif kind == 'compute#region': project_id = self_link[-3] region = self_link[-1] @@ -70,3 +74,7 @@ def start_discovery(resources, response=None): else: for result in _handle_discovery(resources, response): yield result + per_network_quota = resources['config:custom_quota'].get('networks', {}) + for network_id, overrides in per_network_quota.items(): + quota = {k: {'limit': v} for k, v in overrides.items()} + yield Resource(NAME, network_id, quota) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py index 2d38a22fa8..99f92ec2c2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py @@ -22,6 +22,7 @@ @register_timeseries def timeseries(resources): + 'Derive network timeseries for firewall policies.' LOGGER.info('timeseries') for v in resources['firewall_policies'].values(): tuples = int(v['num_tuples']) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index caf41aca2c..ad18f2db94 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -22,6 +22,7 @@ @register_timeseries def timeseries(resources): + 'Derive and yield network and project timeseries for firewall rules.' LOGGER.info('timeseries') grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['network']) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index df580906d1..467c907763 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -30,6 +30,7 @@ def _group_timeseries(name, resources, grouped, limit_name): + 'Derive and yield timeseries from grouped iterators and limits.' for network_id, elements in grouped: network = resources['networks'].get(network_id) if not network: @@ -45,6 +46,7 @@ def _group_timeseries(name, resources, grouped, limit_name): def _forwarding_rules(resources): + 'Derive network timeseries for forwarding rule utilization.' filter = lambda n, v: v['load_balancing_scheme'] != n forwarding_rules = resources['forwarding_rules'].values() forwarding_rules_l4 = itertools.filterfalse( @@ -62,6 +64,7 @@ def _forwarding_rules(resources): def _instances(resources): + 'Derive network timeseries for instance utilization.' instance_networks = functools.reduce( operator.add, [i['networks'] for i in resources['instances'].values()]) grouped = itertools.groupby(instance_networks, lambda i: i['network']) @@ -69,28 +72,8 @@ def _instances(resources): 'INSTANCES_PER_NETWORK_GLOBAL') -def _routes(resources): - filter = lambda v: v['next_hop_type'] in ('peering', 'network') - grouped = itertools.groupby(resources['routes'].values(), - lambda v: v['network']) - project_counts = {} - for network_id, elements in grouped: - network = resources['networks'].get(network_id) - count = len(list(elements)) - labels = {'project': network['project_id'], 'network': network['name']} - yield TimeSeries('network/routes_static_used', count, labels) - project_counts[network['project_id']] = project_counts.get( - network['project_id'], 0) + count - for project_id, count in project_counts.items(): - labels = {'project': project_id} - quota = resources['quota'][project_id] - limit = quota.get('ROUTES', LIMITS['ROUTES']) - yield TimeSeries('project/static_routes_used', count, labels) - yield TimeSeries('project/static_routes_available', limit, labels) - yield TimeSeries('project/static_routes_used_ratio', count / limit, labels) - - def _subnet_ranges(resources): + 'Derive network timeseries for subnet range utilization.' grouped = itertools.groupby(resources['subnetworks'].values(), lambda v: v['network']) return _group_timeseries('subnet', resources, grouped, @@ -99,6 +82,7 @@ def _subnet_ranges(resources): @register_timeseries def timeseries(resources): + 'Yield timeseries.' LOGGER.info('timeseries') return itertools.chain(_forwarding_rules(resources), _instances(resources), - _routes(resources), _subnet_ranges(resources)) + _subnet_ranges(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py new file mode 100644 index 0000000000..40679e6ecf --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py @@ -0,0 +1,55 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging + +from . import TimeSeries, register_timeseries + +LIMITS = { + 'PEERINGS_PER_NETWORK': 25, + 'INTERNAL_FORWARDING_RULES_PER_NETWORK': 500, + 'SUBNET_RANGES_PER_NETWORK': 300, + 'DYNAMIC_ROUTES_PER_PEERING_GROUP': 300, + 'STATIC_ROUTES_PER_PEERING_GROUP': 300, + 'INSTANCES_PER_NETWORK_GLOBAL': 15000 +} +LOGGER = logging.getLogger('net-dash.timeseries.peerings') + +# def _static(resources): +# 'Derive network and project timeseries for static routes.' +# filter = lambda v: v['next_hop_type'] in ('peering', 'network') +# routes = itertools.filterfalse(filter, resources['routes'].values()) +# grouped = itertools.groupby(routes, lambda v: v['network']) +# project_counts = {} +# for network_id, elements in grouped: +# network = resources['networks'].get(network_id) +# count = len(list(elements)) +# labels = {'project': network['project_id'], 'network': network['name']} +# yield TimeSeries('network/routes_static_used', count, labels) +# project_counts[network['project_id']] = project_counts.get( +# network['project_id'], 0) + count +# for project_id, count in project_counts.items(): +# labels = {'project': project_id} +# quota = resources['quota'][project_id] +# limit = quota.get('ROUTES', LIMITS['ROUTES']) +# yield TimeSeries('project/routes_static_used', count, labels) +# yield TimeSeries('project/routes_static_available', limit, labels) +# yield TimeSeries('project/routes_static_used_ratio', count / limit, labels) + +# @register_timeseries +# def timeseries(resources): +# 'Yield timeseries.' +# LOGGER.info('timeseries') +# return itertools.chain(_static(resources), _dynamic(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py index 2209aae0a4..33da985797 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools import itertools import logging -import operator from . import TimeSeries, register_timeseries @@ -24,6 +22,7 @@ def _dynamic(resources): + 'Derive network timeseries for dynamic routes.' for network_id, router_counts in resources['routes_dynamic'].items(): network = resources['networks'][network_id] count = sum(router_counts.values()) @@ -35,6 +34,7 @@ def _dynamic(resources): def _static(resources): + 'Derive network and project timeseries for static routes.' filter = lambda v: v['next_hop_type'] in ('peering', 'network') routes = itertools.filterfalse(filter, resources['routes'].values()) grouped = itertools.groupby(routes, lambda v: v['network']) @@ -57,5 +57,6 @@ def _static(resources): @register_timeseries def timeseries(resources): + 'Yield timeseries.' LOGGER.info('timeseries') return itertools.chain(_static(resources), _dynamic(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 2697d5a00c..41c896a107 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -59,6 +59,7 @@ def _subnet_instances(resources): @register_timeseries def timeseries(resources): + 'Derive and yield subnetwork timeseries for address utilization.' LOGGER.info('timeseries') subnet_nets = { k: ipaddress.ip_network(v['cidr_range']) From 1530080572e5faef9559c413247e7d5ca23ee629 Mon Sep 17 00:00:00 2001 From: Ludo Date: Wed, 23 Nov 2022 08:09:10 +0100 Subject: [PATCH 34/82] simpler quota, network peering timeseries --- .../network-dashboard/cf/custom-quotas.sample | 3 +- .../network-dashboard/cf/out.json | 2706 ++++------------- .../cf/plugins/discover-cai-compute.py | 3 +- .../cf/plugins/discover-compute-quota.py | 17 +- .../cf/plugins/series-firewall-rules.py | 2 +- .../cf/plugins/series-networks.py | 21 +- ...s-peerings.py => series-peering-groups.py} | 12 +- .../cf/plugins/series-routes.py | 2 +- 8 files changed, 608 insertions(+), 2158 deletions(-) rename blueprints/cloud-operations/network-dashboard/cf/plugins/{series-peerings.py => series-peering-groups.py} (92%) diff --git a/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample b/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample index 61085d2b30..9f090b3c50 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample +++ b/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample @@ -1,6 +1,7 @@ projects: tf-playground-svpc-net: - INTERNAL_FORWARDING_RULES_PER_NETWORK: 750 + global: + INTERNAL_FORWARDING_RULES_PER_NETWORK: 750 networks: # TODO: what are the quotas that can be overridden at the network level? projects/tf-playground-svpc-net/global/networks/shared-vpc: diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index a266d50ecc..de591f19e2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -12,7 +12,9 @@ "config:custom_quota": { "projects": { "tf-playground-svpc-net": { - "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 + "global": { + "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 + } } }, "networks": { @@ -527,16 +529,24 @@ "project_number": "195159130008", "peerings": [ { + "active": true, "name": "prod-peering-0-prod-spoke-0-prod-landing-0", "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" }, { + "active": true, "name": "servicenetworking-googleapis-com", "network": "projects/r63cc730008bdfb49p-tp/global/networks/servicenetworking" }, { + "active": true, "name": "to-dev-spoke-0", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + }, + { + "active": false, + "name": "to-test-public-gcs", + "network": "projects/186042149622/global/networks/default" } ], "subnetworks": [ @@ -554,10 +564,12 @@ "project_number": "233262889141", "peerings": [ { + "active": true, "name": "dev-peering-0-prod-landing-0-dev-spoke-0", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" }, { + "active": true, "name": "prod-peering-0-prod-landing-0-prod-spoke-0", "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" } @@ -595,14 +607,32 @@ "project_number": "759592912116", "peerings": [ { + "active": false, + "name": "composer-474420912893-vpc-peering", + "network": "projects/474420912893/global/networks/composer-net" + }, + { + "active": false, + "name": "composer-566526877645-vpc-peering", + "network": "projects/566526877645/global/networks/composer-net" + }, + { + "active": true, "name": "dev-peering-0-dev-spoke-0-prod-landing-0", "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" }, { + "active": false, + "name": "gke-n0a46a71e52c28142071-9ca9-f42d-peer", + "network": "projects/gke-prod-europe-west1-b-4/global/networks/gke-n0a46a71e52c28142071-9ca9-32cb-net" + }, + { + "active": true, "name": "servicenetworking-googleapis-com", "network": "projects/p10d8c97c9bd39799p-tp/global/networks/servicenetworking" }, { + "active": true, "name": "to-prod-spoke-0", "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" } @@ -624,6 +654,7 @@ "project_number": "1079408472053", "peerings": [ { + "active": true, "name": "servicenetworking-googleapis-com", "network": "projects/a2fed18bfde5785bdp-tp/global/networks/servicenetworking" } @@ -2012,2195 +2043,592 @@ "quota": { "tf-playground-simple": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 2 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 2 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 1 - }, - "NETWORKS": { - "limit": 15, - "usage": 2 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 2 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 1 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "ludo-dev-net-spoke-0": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 5 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 1 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 1 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 2 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 4 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "ludo-prod-net-landing-0": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 1 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 1 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 1 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "ludo-prod-net-spoke-0": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 1 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 1 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 1 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "ludo-dev-sec-core-0": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "ludo-prod-sec-core-0": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-gcs-test-0": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-svpc-gce-dr": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-svpc-net-dr": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 4 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 1 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 2 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 2 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-svpc-openshift": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-svpc-gce": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 9, - "usage": 1 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 1 - }, - "IMAGES": { - "limit": 2000, - "usage": 1 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 9, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-svpc-net": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 1 - }, - "FIREWALLS": { - "limit": 200, - "usage": 9 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 3 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 1 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 1 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 1 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 1 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 6 - }, - "ROUTES": { - "limit": 250, - "usage": 3 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 2 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 1 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 1 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 1 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 1 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 1 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 2 - }, - "INTERNAL_FORWARDING_RULES_PER_NETWORK": { - "limit": 750 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000, + "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 } }, "tf-playground-svpc-gke": { "global": { - "BACKEND_BUCKETS": { - "limit": 9, - "usage": 0 - }, - "BACKEND_SERVICES": { - "limit": 75, - "usage": 0 - }, - "EXTERNAL_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "FIREWALLS": { - "limit": 200, - "usage": 0 - }, - "FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "GLOBAL_INTERNAL_ADDRESSES": { - "limit": 5000, - "usage": 0 - }, - "HEALTH_CHECKS": { - "limit": 150, - "usage": 0 - }, - "IMAGES": { - "limit": 2000, - "usage": 0 - }, - "INSTANCE_TEMPLATES": { - "limit": 300, - "usage": 0 - }, - "INTERCONNECTS": { - "limit": 6, - "usage": 0 - }, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": { - "limit": 45, - "usage": 0 - }, - "IN_USE_ADDRESSES": { - "limit": 69, - "usage": 0 - }, - "MACHINE_IMAGES": { - "limit": 2000, - "usage": 0 - }, - "NETWORKS": { - "limit": 15, - "usage": 0 - }, - "NETWORK_ENDPOINT_GROUPS": { - "limit": 300, - "usage": 0 - }, - "NETWORK_FIREWALL_POLICIES": { - "limit": 30, - "usage": 0 - }, - "PACKET_MIRRORINGS": { - "limit": 45, - "usage": 0 - }, - "PUBLIC_ADVERTISED_PREFIXES": { - "limit": 1, - "usage": 0 - }, - "PUBLIC_DELEGATED_PREFIXES": { - "limit": 10, - "usage": 0 - }, - "ROUTERS": { - "limit": 10, - "usage": 0 - }, - "ROUTES": { - "limit": 250, - "usage": 0 - }, - "SECURITY_POLICIES": { - "limit": 10, - "usage": 0 - }, - "SECURITY_POLICY_CEVAL_RULES": { - "limit": 20, - "usage": 0 - }, - "SECURITY_POLICY_RULES": { - "limit": 100, - "usage": 0 - }, - "SNAPSHOTS": { - "limit": 5000, - "usage": 0 - }, - "SSL_CERTIFICATES": { - "limit": 30, - "usage": 0 - }, - "STATIC_ADDRESSES": { - "limit": 21, - "usage": 0 - }, - "STATIC_BYOIP_ADDRESSES": { - "limit": 1024, - "usage": 0 - }, - "SUBNETWORKS": { - "limit": 175, - "usage": 0 - }, - "TARGET_HTTPS_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_HTTP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_INSTANCES": { - "limit": 150, - "usage": 0 - }, - "TARGET_POOLS": { - "limit": 150, - "usage": 0 - }, - "TARGET_SSL_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_TCP_PROXIES": { - "limit": 30, - "usage": 0 - }, - "TARGET_VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "URL_MAPS": { - "limit": 30, - "usage": 0 - }, - "VPN_GATEWAYS": { - "limit": 15, - "usage": 0 - }, - "VPN_TUNNELS": { - "limit": 30, - "usage": 0 - }, - "XPN_SERVICE_PROJECTS": { - "limit": 1000, - "usage": 0 - } + "BACKEND_BUCKETS": 9, + "BACKEND_SERVICES": 75, + "EXTERNAL_VPN_GATEWAYS": 15, + "FIREWALLS": 200, + "FORWARDING_RULES": 45, + "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, + "GLOBAL_INTERNAL_ADDRESSES": 5000, + "HEALTH_CHECKS": 150, + "IMAGES": 2000, + "INSTANCE_TEMPLATES": 300, + "INTERCONNECTS": 6, + "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, + "IN_USE_ADDRESSES": 69, + "MACHINE_IMAGES": 2000, + "NETWORKS": 15, + "NETWORK_ENDPOINT_GROUPS": 300, + "NETWORK_FIREWALL_POLICIES": 30, + "PACKET_MIRRORINGS": 45, + "PUBLIC_ADVERTISED_PREFIXES": 1, + "PUBLIC_DELEGATED_PREFIXES": 10, + "ROUTERS": 10, + "ROUTES": 250, + "SECURITY_POLICIES": 10, + "SECURITY_POLICY_CEVAL_RULES": 20, + "SECURITY_POLICY_RULES": 100, + "SNAPSHOTS": 5000, + "SSL_CERTIFICATES": 30, + "STATIC_ADDRESSES": 21, + "STATIC_BYOIP_ADDRESSES": 1024, + "SUBNETWORKS": 175, + "TARGET_HTTPS_PROXIES": 30, + "TARGET_HTTP_PROXIES": 30, + "TARGET_INSTANCES": 150, + "TARGET_POOLS": 150, + "TARGET_SSL_PROXIES": 30, + "TARGET_TCP_PROXIES": 30, + "TARGET_VPN_GATEWAYS": 15, + "URL_MAPS": 30, + "VPN_GATEWAYS": 15, + "VPN_TUNNELS": 30, + "XPN_SERVICE_PROJECTS": 1000 } }, "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "PEERINGS_PER_NETWORK": { - "limit": 40 - } + "PEERINGS_PER_NETWORK": 40 } }, "routes_dynamic": { diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index b156063143..519274c645 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -132,9 +132,10 @@ def _handle_instances(resource, data): def _handle_networks(resource, data): 'Handle network type resource data.' peerings = [{ + 'active': p['state'] == 'ACTIVE', 'name': p['name'], 'network': _self_link(p['network']) - } for p in data.get('peerings', []) if p['state'] == 'ACTIVE'] + } for p in data.get('peerings', [])] subnets = [_self_link(s) for s in data.get('subnetworks', [])] return {'peerings': peerings, 'subnetworks': subnets} diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 48abbf7558..093cd3d520 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -31,10 +31,8 @@ def _handle_discovery(resources, response): for part in dirty_mp_response(content_type, response.content): kind = part.get('kind') quota = { - q['metric']: { - 'limit': q['limit'], - 'usage': q['usage'] - } for q in sorted(part.get('quotas', []), key=lambda v: v['metric']) + q['metric']: int(q['limit']) + for q in sorted(part.get('quotas', []), key=lambda v: v['metric']) } self_link = part.get('selfLink') if not self_link: @@ -43,12 +41,12 @@ def _handle_discovery(resources, response): if kind == 'compute#project': project_id = self_link[-1] region = 'global' - for k, v in per_project_quota.get(project_id, {}).items(): - metric = quota.setdefault(k, {}) - metric['limit'] = v elif kind == 'compute#region': project_id = self_link[-3] region = self_link[-1] + # custom quota overrides + for k, v in per_project_quota.get(project_id, {}).get(region, {}).items(): + quota[k] = int(v) if project_id not in resources[NAME]: resources[NAME][project_id] = {} yield Resource(NAME, project_id, quota, region) @@ -56,6 +54,7 @@ def _handle_discovery(resources, response): @register_init def init(resources): + 'Create the quota key in the shared resource map.' LOGGER.info('init') if NAME not in resources: resources[NAME] = {} @@ -63,6 +62,7 @@ def init(resources): @register_discovery(Level.DERIVED, 0) def start_discovery(resources, response=None): + 'Fetch and process quota for projects.' LOGGER.info(f'discovery (has response: {response is not None})') if response is None: # TODO: regions @@ -74,7 +74,8 @@ def start_discovery(resources, response=None): else: for result in _handle_discovery(resources, response): yield result + # store custom network-level quota per_network_quota = resources['config:custom_quota'].get('networks', {}) for network_id, overrides in per_network_quota.items(): - quota = {k: {'limit': v} for k, v in overrides.items()} + quota = {k: int(v) for k, v in overrides.items()} yield Resource(NAME, network_id, quota) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index ad18f2db94..5d8303408a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -37,7 +37,7 @@ def timeseries(resources): lambda v: v['project_id']) for project_id, rules in grouped: count = len(list(rules)) - limit = int(resources['quota'][project_id]['global']['FIREWALLS']['limit']) + limit = int(resources['quota'][project_id]['global']['FIREWALLS']) labels = {'project_id': project_id} yield TimeSeries('project/firewall_rules_used', count, labels) yield TimeSeries('project/firewalls_rules_available', limit, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index 467c907763..852f9958f7 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -38,7 +38,7 @@ def _group_timeseries(name, resources, grouped, limit_name): continue count = len(list(elements)) labels = {'project': network['project_id'], 'network': network['name']} - quota = resources['quota'][network['project_id']] + quota = resources['quota'][network['project_id']]['global'] limit = quota.get(limit_name, LIMITS[limit_name]) yield TimeSeries(f'network/{name}_used', count, labels) yield TimeSeries(f'network/{name}_available', limit, labels) @@ -72,6 +72,23 @@ def _instances(resources): 'INSTANCES_PER_NETWORK_GLOBAL') +def _peerings(resources): + quota = resources['quota'] + for network_id, network in resources['networks'].items(): + labels = {'project': network['project_id'], 'network': network['name']} + limit = quota.get(network_id, {}).get('PEERINGS_PER_NETWORK', 250) + p_active = len([p for p in network['peerings'] if p['active']]) + p_total = len(network['peerings']) + yield TimeSeries('network/peering_active_used', p_active, labels) + yield TimeSeries('network/peering_active_available', limit, labels) + yield TimeSeries('network/peering_active_used_ratio', p_active / limit, + labels) + yield TimeSeries('network/peering_total_used', p_total, labels) + yield TimeSeries('network/peering_total_available', limit, labels) + yield TimeSeries('network/peering_total_used_ratio', p_total / limit, + labels) + + def _subnet_ranges(resources): 'Derive network timeseries for subnet range utilization.' grouped = itertools.groupby(resources['subnetworks'].values(), @@ -85,4 +102,4 @@ def timeseries(resources): 'Yield timeseries.' LOGGER.info('timeseries') return itertools.chain(_forwarding_rules(resources), _instances(resources), - _subnet_ranges(resources)) + _peerings(resources), _subnet_ranges(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py similarity index 92% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py index 40679e6ecf..9bbc6e5fe3 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peerings.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py @@ -48,8 +48,10 @@ # yield TimeSeries('project/routes_static_available', limit, labels) # yield TimeSeries('project/routes_static_used_ratio', count / limit, labels) -# @register_timeseries -# def timeseries(resources): -# 'Yield timeseries.' -# LOGGER.info('timeseries') -# return itertools.chain(_static(resources), _dynamic(resources)) + +@register_timeseries +def timeseries(resources): + 'Yield timeseries.' + LOGGER.info('timeseries') + return + yield diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py index 33da985797..3502887fb8 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py @@ -48,7 +48,7 @@ def _static(resources): network['project_id'], 0) + count for project_id, count in project_counts.items(): labels = {'project': project_id} - quota = resources['quota'][project_id] + quota = resources['quota'][project_id]['global'] limit = quota.get('ROUTES', LIMITS['ROUTES']) yield TimeSeries('project/routes_static_used', count, labels) yield TimeSeries('project/routes_static_available', limit, labels) From 25577aa2fb5b2d765e48409d02870d6da2ea664e Mon Sep 17 00:00:00 2001 From: Ludo Date: Wed, 23 Nov 2022 19:36:07 +0100 Subject: [PATCH 35/82] peering timeseries --- .../network-dashboard/cf/out.json | 39 ++++-- .../cf/plugins/discover-cai-compute.py | 3 +- .../cf/plugins/series-peering-groups.py | 131 ++++++++++++++---- 3 files changed, 131 insertions(+), 42 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index de591f19e2..740a0fc4de 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -531,22 +531,26 @@ { "active": true, "name": "prod-peering-0-prod-spoke-0-prod-landing-0", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", + "project_id": "ludo-prod-net-landing-0" }, { "active": true, "name": "servicenetworking-googleapis-com", - "network": "projects/r63cc730008bdfb49p-tp/global/networks/servicenetworking" + "network": "projects/r63cc730008bdfb49p-tp/global/networks/servicenetworking", + "project_id": "r63cc730008bdfb49p-tp" }, { "active": true, "name": "to-dev-spoke-0", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "project_id": "ludo-dev-net-spoke-0" }, { "active": false, "name": "to-test-public-gcs", - "network": "projects/186042149622/global/networks/default" + "network": "projects/186042149622/global/networks/default", + "project_id": "186042149622" } ], "subnetworks": [ @@ -566,12 +570,14 @@ { "active": true, "name": "dev-peering-0-prod-landing-0-dev-spoke-0", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" + "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", + "project_id": "ludo-dev-net-spoke-0" }, { "active": true, "name": "prod-peering-0-prod-landing-0-prod-spoke-0", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "project_id": "ludo-prod-net-spoke-0" } ], "subnetworks": [ @@ -609,32 +615,38 @@ { "active": false, "name": "composer-474420912893-vpc-peering", - "network": "projects/474420912893/global/networks/composer-net" + "network": "projects/474420912893/global/networks/composer-net", + "project_id": "474420912893" }, { "active": false, "name": "composer-566526877645-vpc-peering", - "network": "projects/566526877645/global/networks/composer-net" + "network": "projects/566526877645/global/networks/composer-net", + "project_id": "566526877645" }, { "active": true, "name": "dev-peering-0-dev-spoke-0-prod-landing-0", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" + "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", + "project_id": "ludo-prod-net-landing-0" }, { "active": false, "name": "gke-n0a46a71e52c28142071-9ca9-f42d-peer", - "network": "projects/gke-prod-europe-west1-b-4/global/networks/gke-n0a46a71e52c28142071-9ca9-32cb-net" + "network": "projects/gke-prod-europe-west1-b-4/global/networks/gke-n0a46a71e52c28142071-9ca9-32cb-net", + "project_id": "gke-prod-europe-west1-b-4" }, { "active": true, "name": "servicenetworking-googleapis-com", - "network": "projects/p10d8c97c9bd39799p-tp/global/networks/servicenetworking" + "network": "projects/p10d8c97c9bd39799p-tp/global/networks/servicenetworking", + "project_id": "p10d8c97c9bd39799p-tp" }, { "active": true, "name": "to-prod-spoke-0", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" + "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", + "project_id": "ludo-prod-net-spoke-0" } ], "subnetworks": [ @@ -656,7 +668,8 @@ { "active": true, "name": "servicenetworking-googleapis-com", - "network": "projects/a2fed18bfde5785bdp-tp/global/networks/servicenetworking" + "network": "projects/a2fed18bfde5785bdp-tp/global/networks/servicenetworking", + "project_id": "a2fed18bfde5785bdp-tp" } ], "subnetworks": [ diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 519274c645..f27343afbc 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -134,7 +134,8 @@ def _handle_networks(resource, data): peerings = [{ 'active': p['state'] == 'ACTIVE', 'name': p['name'], - 'network': _self_link(p['network']) + 'network': _self_link(p['network']), + 'project_id': _self_link(p['network']).split('/')[1] } for p in data.get('peerings', [])] subnets = [_self_link(s) for s in data.get('subnetworks', [])] return {'peerings': peerings, 'subnetworks': subnets} diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py index 9bbc6e5fe3..08ee875408 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py @@ -18,40 +18,115 @@ from . import TimeSeries, register_timeseries LIMITS = { - 'PEERINGS_PER_NETWORK': 25, - 'INTERNAL_FORWARDING_RULES_PER_NETWORK': 500, - 'SUBNET_RANGES_PER_NETWORK': 300, - 'DYNAMIC_ROUTES_PER_PEERING_GROUP': 300, - 'STATIC_ROUTES_PER_PEERING_GROUP': 300, - 'INSTANCES_PER_NETWORK_GLOBAL': 15000 + 'forwarding_rules_l4': { + 'pg': ('INTERNAL_FORWARDING_RULES_PER_PEERING_GROUP', 500), + 'prj': ('INTERNAL_FORWARDING_RULES_PER_NETWORK', 500) + }, + 'forwarding_rules_l7': { + 'pg': ('INTERNAL_MANAGED_FORWARDING_RULES_PER_PEERING_GROUP', 175), + 'prj': ('INTERNAL_MANAGED_FORWARDING_RULES_PER_NETWORK', 75) + }, + 'instances': { + 'pg': ('INSTANCES_PER_PEERING_GROUP', 15500), + 'prj': ('INSTANCES_PER_NETWORK_GLOBAL', 15000) + }, + 'routes': { + 'pg': ('STATIC_ROUTES_PER_PEERING_GROUP', 300), + 'prj': ('ROUTES', 250) + }, + 'routes_dynamic': { + 'pg': ('DYNAMIC_ROUTES_PER_PEERING_GROUP', 300), + 'prj': ('', 100) + } } LOGGER = logging.getLogger('net-dash.timeseries.peerings') -# def _static(resources): -# 'Derive network and project timeseries for static routes.' -# filter = lambda v: v['next_hop_type'] in ('peering', 'network') -# routes = itertools.filterfalse(filter, resources['routes'].values()) -# grouped = itertools.groupby(routes, lambda v: v['network']) -# project_counts = {} -# for network_id, elements in grouped: -# network = resources['networks'].get(network_id) -# count = len(list(elements)) -# labels = {'project': network['project_id'], 'network': network['name']} -# yield TimeSeries('network/routes_static_used', count, labels) -# project_counts[network['project_id']] = project_counts.get( -# network['project_id'], 0) + count -# for project_id, count in project_counts.items(): -# labels = {'project': project_id} -# quota = resources['quota'][project_id] -# limit = quota.get('ROUTES', LIMITS['ROUTES']) -# yield TimeSeries('project/routes_static_used', count, labels) -# yield TimeSeries('project/routes_static_available', limit, labels) -# yield TimeSeries('project/routes_static_used_ratio', count / limit, labels) + +def _count_forwarding_rules_l4(resources, network_ids): + return len([ + r for r in resources['forwarding_rules'].values() if + r['network'] in network_ids and r['load_balancing_scheme'] == 'INTERNAL' + ]) + + +def _count_forwarding_rules_l4(resources, network_ids): + return len([ + r for r in resources['forwarding_rules'].values() + if r['network'] in network_ids and + r['load_balancing_scheme'] == 'INTERNAL_MANAGED' + ]) + + +def _count_instances(resources, network_ids): + count = 0 + for i in resources['instances'].values(): + if any(n['network'] in network_ids for n in i['networks']): + count += 1 + return count + + +def _count_routes(resources, network_ids): + return len( + [r for r in resources['routes'].values() if r['network'] in network_ids]) + + +def _count_routes_dynamic(resources, network_ids): + return sum([ + sum(v.values()) + for k, v in resources['routes_dynamic'].items() + if k in network_ids + ]) + + +def _get_limit_max(quota, network_id, project_id, resource_name): + pg_name, pg_default = LIMITS[resource_name]['pg'] + prj_name, prj_default = LIMITS[resource_name]['prj'] + network_quota = quota.get(network_id, {}) + project_quota = quota.get(project_id, {}).get('global', {}) + return max([ + network_quota.get(pg_name, 0), + project_quota.get(prj_name, prj_default), + project_quota.get(pg_name, pg_default) + ]) + + +def _get_limit(quota, network, resource_name): + # https://cloud.google.com/vpc/docs/quota#vpc-peering-ilb-example + # vpc_max = max(vpc limit, pg limit) + vpc_max = _get_limit_max(quota, network['self_link'], network['project_id'], + resource_name) + # peers_max = [max(vpc limit, pg limit) for v in peered vpcs] + # peers_min = min(peers_max) + peers_min = min([ + _get_limit_max(quota, p['network'], p['project_id'], resource_name) + for p in network['peerings'] + ]) + # max(vpc_max, peers_min) + return max([vpc_max, peers_min]) + + +def _network_timeseries(resources, network): + if len(network['peerings']) == 0: + return + network_ids = [network['self_link'] + ] + [p['network'] for p in network['peerings']] + for resource_name in LIMITS: + limit = _get_limit(resources['quota'], network, resource_name) + func = globals().get(f'_count_{resource_name}') + if not func or not callable(func): + LOGGER.critical(f'no handler for {resource_name} or handler not callable') + continue + count = func(resources, network_ids) + labels = {'project': network['project_id'], 'network': network['name']} + yield TimeSeries(f'network/{resource_name}_used', count, labels) + yield TimeSeries(f'network/{resource_name}_available', limit, labels) + yield TimeSeries(f'network/{resource_name}_used_ratio', count / limit, + labels) @register_timeseries def timeseries(resources): 'Yield timeseries.' LOGGER.info('timeseries') - return - yield + return itertools.chain(*(_network_timeseries(resources, n) + for n in resources['networks'].values())) From 3ca8d975ce663e86f46f4f7d36cf57f4557d7eac Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 24 Nov 2022 01:37:32 +0100 Subject: [PATCH 36/82] timeseries names --- .../network-dashboard/cf/NOTES.md | 2 +- .../network-dashboard/cf/main.py | 3 +- .../cf/plugins/descriptors.yaml | 49 +++++++++++++++++++ .../cf/plugins/series-firewall-policies.py | 8 +-- .../cf/plugins/series-firewall-rules.py | 7 ++- .../cf/plugins/series-networks.py | 18 +++---- .../cf/plugins/series-peering-groups.py | 12 ++--- 7 files changed, 73 insertions(+), 26 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md index ab1dd68cfb..c73ade11b2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md +++ b/blueprints/cloud-operations/network-dashboard/cf/NOTES.md @@ -60,7 +60,7 @@ `routes.get_static_routes_data` - [x] calculate and store peering metrics `peerings.get_vpc_peering_data` -- [ ] calculate and store peering group metrics +- [x] calculate and store peering group metrics `metrics.get_pgg_data` `routes.get_routes_ppg` - [ ] write buffered timeseries diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index bf81019505..942ca7028e 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -141,8 +141,7 @@ def main(organization=None, op_project=None, project=None, folder=None, if dump_file: json.dump(resources, dump_file, indent=2) - from icecream import ic - ic(timeseries) + print('\n'.join(sorted(list(set(t.metric for t in timeseries))))) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml b/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml new file mode 100644 index 0000000000..dd4a69738e --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml @@ -0,0 +1,49 @@ +prefix: net-dash/ +descriptors: + firewall_policies/tuples_available: Firewall policy tuples available + firewall_policies/tuples_used: Firewall policy tuples used + firewall_policies/tuples_used_ratio: Firewall policy tuples used ratio + network/firewall_rules/used: Firewall rules used + network/forwarding_rules_l7_available: + network/forwarding_rules_l7_used: + network/forwarding_rules_l7_used_ratio: + network/instances_available: + network/instances_used: + network/instances_used_ratio: + network/peerings_active_available: + network/peerings_active_used: + network/peerings_active_used_ratio: + network/peerings_total_available: + network/peerings_total_used: + network/peerings_total_used_ratio: + network/routes_dynamic_available: + network/routes_dynamic_used: + network/routes_dynamic_used_ratio: + network/routes_static_used: + network/subnets_available: + network/subnets_used: + network/subnets_used_ratio: + peering_group/forwarding_rules_l4_available: + peering_group/forwarding_rules_l4_used: + peering_group/forwarding_rules_l4_used_ratio: + peering_group/forwarding_rules_l7_available: + peering_group/forwarding_rules_l7_used: + peering_group/forwarding_rules_l7_used_ratio: + peering_group/instances_available: + peering_group/instances_used: + peering_group/instances_used_ratio: + peering_group/routes_dynamic_available: + peering_group/routes_dynamic_used: + peering_group/routes_dynamic_used_ratio: + peering_group/routes_static_available: + peering_group/routes_static_used: + peering_group/routes_static_used_ratio: + project/firewall_rules_available: + project/firewall_rules_used: + project/firewall_rules_used_ratio: + project/routes_static_available: + project/routes_static_used: + project/routes_static_used_ratio: + subnetwork/addresses_available: + subnetwork/addresses_used: + subnetwork/addresses_used_ratio: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py index 99f92ec2c2..4534ba1984 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py @@ -27,7 +27,7 @@ def timeseries(resources): for v in resources['firewall_policies'].values(): tuples = int(v['num_tuples']) labels = {'parent': v['parent'], 'name': v['name']} - yield TimeSeries('firewall_policy/tuples_used', tuples, labels) - yield TimeSeries('firewall_policy/tuples_available', TUPLE_LIMIT, labels) - yield TimeSeries('firewalls_policy/tuples_used_ratio', tuples / TUPLE_LIMIT, - labels) + yield TimeSeries('firewall_policies/tuples_used', tuples, labels) + yield TimeSeries('firewall_policies/tuples_available', TUPLE_LIMIT, labels) + yield TimeSeries('firewall_policies/tuples_used_ratio', + tuples / TUPLE_LIMIT, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index 5d8303408a..7c967ef759 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -32,7 +32,7 @@ def timeseries(resources): 'network_name': resources['networks'][network_id]['name'], 'project_id': resources['networks'][network_id]['project_id'] } - yield TimeSeries('network/firewalls/used', count, labels) + yield TimeSeries('network/firewall_rules/used', count, labels) grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['project_id']) for project_id, rules in grouped: @@ -40,6 +40,5 @@ def timeseries(resources): limit = int(resources['quota'][project_id]['global']['FIREWALLS']) labels = {'project_id': project_id} yield TimeSeries('project/firewall_rules_used', count, labels) - yield TimeSeries('project/firewalls_rules_available', limit, labels) - yield TimeSeries('project/firewalls_rules_used_ratio', count / limit, - labels) + yield TimeSeries('project/firewall_rules_available', limit, labels) + yield TimeSeries('project/firewall_rules_used_ratio', count / limit, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index 852f9958f7..a789c29c6e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -56,9 +56,9 @@ def _forwarding_rules(resources): grouped_l4 = itertools.groupby(forwarding_rules_l4, lambda i: i['network']) grouped_l7 = itertools.groupby(forwarding_rules_l7, lambda i: i['network']) return itertools.chain( - _group_timeseries('forwarding_rule_l4', resources, grouped_l4, + _group_timeseries('forwarding_rules_l4', resources, grouped_l4, 'INTERNAL_FORWARDING_RULES_PER_NETWORK'), - _group_timeseries('forwarding_rule_l7', resources, grouped_l7, + _group_timeseries('forwarding_rules_l7', resources, grouped_l7, 'INTERNAL_MANAGED_FORWARDING_RULES_PER_NETWORK'), ) @@ -79,13 +79,13 @@ def _peerings(resources): limit = quota.get(network_id, {}).get('PEERINGS_PER_NETWORK', 250) p_active = len([p for p in network['peerings'] if p['active']]) p_total = len(network['peerings']) - yield TimeSeries('network/peering_active_used', p_active, labels) - yield TimeSeries('network/peering_active_available', limit, labels) - yield TimeSeries('network/peering_active_used_ratio', p_active / limit, + yield TimeSeries('network/peerings_active_used', p_active, labels) + yield TimeSeries('network/peerings_active_available', limit, labels) + yield TimeSeries('network/peerings_active_used_ratio', p_active / limit, labels) - yield TimeSeries('network/peering_total_used', p_total, labels) - yield TimeSeries('network/peering_total_available', limit, labels) - yield TimeSeries('network/peering_total_used_ratio', p_total / limit, + yield TimeSeries('network/peerings_total_used', p_total, labels) + yield TimeSeries('network/peerings_total_available', limit, labels) + yield TimeSeries('network/peerings_total_used_ratio', p_total / limit, labels) @@ -93,7 +93,7 @@ def _subnet_ranges(resources): 'Derive network timeseries for subnet range utilization.' grouped = itertools.groupby(resources['subnetworks'].values(), lambda v: v['network']) - return _group_timeseries('subnet', resources, grouped, + return _group_timeseries('subnets', resources, grouped, 'SUBNET_RANGES_PER_NETWORK') diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py index 08ee875408..758ecbfb42 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py @@ -30,7 +30,7 @@ 'pg': ('INSTANCES_PER_PEERING_GROUP', 15500), 'prj': ('INSTANCES_PER_NETWORK_GLOBAL', 15000) }, - 'routes': { + 'routes_static': { 'pg': ('STATIC_ROUTES_PER_PEERING_GROUP', 300), 'prj': ('ROUTES', 250) }, @@ -49,7 +49,7 @@ def _count_forwarding_rules_l4(resources, network_ids): ]) -def _count_forwarding_rules_l4(resources, network_ids): +def _count_forwarding_rules_l7(resources, network_ids): return len([ r for r in resources['forwarding_rules'].values() if r['network'] in network_ids and @@ -65,7 +65,7 @@ def _count_instances(resources, network_ids): return count -def _count_routes(resources, network_ids): +def _count_routes_static(resources, network_ids): return len( [r for r in resources['routes'].values() if r['network'] in network_ids]) @@ -118,9 +118,9 @@ def _network_timeseries(resources, network): continue count = func(resources, network_ids) labels = {'project': network['project_id'], 'network': network['name']} - yield TimeSeries(f'network/{resource_name}_used', count, labels) - yield TimeSeries(f'network/{resource_name}_available', limit, labels) - yield TimeSeries(f'network/{resource_name}_used_ratio', count / limit, + yield TimeSeries(f'peering_group/{resource_name}_used', count, labels) + yield TimeSeries(f'peering_group/{resource_name}_available', limit, labels) + yield TimeSeries(f'peering_group/{resource_name}_used_ratio', count / limit, labels) From 678fdf0028d71bb7078309a7c9491e1fe072e57d Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 24 Nov 2022 10:48:25 +0100 Subject: [PATCH 37/82] wip descriptors --- .../network-dashboard/cf/main.py | 22 ++++- .../network-dashboard/cf/plugins/__init__.py | 2 + .../cf/plugins/descriptors.yaml | 94 +++++++++---------- .../cf/plugins/series-firewall-policies.py | 12 ++- .../cf/plugins/series-firewall-rules.py | 18 +++- .../cf/plugins/series-networks.py | 27 +++++- 6 files changed, 118 insertions(+), 57 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 942ca7028e..dc4a554531 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -31,6 +31,7 @@ def do_discovery(resources): + 'Discover resources needed in measurements.' LOGGER.info(f'discovery start') for plugin in plugins.get_discovery_plugins(): q = collections.deque(plugin.func(resources)) @@ -59,6 +60,7 @@ def do_discovery(resources): def do_init(resources, organization, op_project, folders=None, projects=None, custom_quota=None): + 'Prepare the resources datastructure fields.' LOGGER.info(f'init start') resources['config:organization'] = str(organization) resources['config:monitoring_project'] = op_project @@ -69,7 +71,13 @@ def do_init(resources, organization, op_project, folders=None, projects=None, plugin.func(resources) -def do_timeseries(resources, timeseries, debug_plugin=None): +def do_metric_descriptors(timeseries, op_project): + 'Create missing descriptors for custom metrics in timeseries.' + pass + + +def do_timeseries(resources, descriptors, timeseries, debug_plugin=None): + 'Create timeseries.' LOGGER.info(f'timeseries start (debug plugin: {debug_plugin})') for plugin in plugins.get_timeseries_plugins(): if debug_plugin and plugin.name != debug_plugin: @@ -78,10 +86,14 @@ def do_timeseries(resources, timeseries, debug_plugin=None): for result in plugin.func(resources): if not result: continue - timeseries.append(result) + if isinstance(result, plugins.MetricDescriptor): + descriptors[result.type] = result + elif isinstance(result, plugins.TimeSeries): + timeseries.append(result) def fetch(request): + 'Minimal HTTP client interface for API calls.' # try LOGGER.info(f'fetch {request.url}') try: @@ -120,6 +132,7 @@ def main(organization=None, op_project=None, project=None, folder=None, custom_quota_file=None, dump_file=None, load_file=None, debug_plugin=None): logging.basicConfig(level=logging.INFO) + descriptors = {} timeseries = [] if load_file: resources = json.load(load_file) @@ -133,7 +146,7 @@ def main(organization=None, op_project=None, project=None, folder=None, raise SystemExit(f'Error decoding custom quota file: {e.args[0]}') do_init(resources, organization, op_project, folder, project, custom_quota) do_discovery(resources) - do_timeseries(resources, timeseries, debug_plugin) + do_timeseries(resources, descriptors, timeseries, debug_plugin) LOGGER.info( {k: len(v) for k, v in resources.items() if not isinstance(v, str)}) LOGGER.info(f'{len(timeseries)} timeseries') @@ -141,7 +154,8 @@ def main(organization=None, op_project=None, project=None, folder=None, if dump_file: json.dump(resources, dump_file, indent=2) - print('\n'.join(sorted(list(set(t.metric for t in timeseries))))) + import icecream + icecream.ic(descriptors) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index cc4b77131d..d4f444e0b3 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -32,6 +32,8 @@ HTTPRequest = collections.namedtuple('HTTPRequest', 'url headers data json', defaults=[True]) Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') +MetricDescriptor = collections.namedtuple('MetricDescriptor', + 'type name labels is_ratio') Plugin = collections.namedtuple('Plugin', 'func name level priority', defaults=[Level.PRIMARY, 99]) Resource = collections.namedtuple('Resource', 'type id data key', diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml b/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml index dd4a69738e..c03a37bf91 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml @@ -1,49 +1,49 @@ prefix: net-dash/ descriptors: - firewall_policies/tuples_available: Firewall policy tuples available - firewall_policies/tuples_used: Firewall policy tuples used - firewall_policies/tuples_used_ratio: Firewall policy tuples used ratio - network/firewall_rules/used: Firewall rules used - network/forwarding_rules_l7_available: - network/forwarding_rules_l7_used: - network/forwarding_rules_l7_used_ratio: - network/instances_available: - network/instances_used: - network/instances_used_ratio: - network/peerings_active_available: - network/peerings_active_used: - network/peerings_active_used_ratio: - network/peerings_total_available: - network/peerings_total_used: - network/peerings_total_used_ratio: - network/routes_dynamic_available: - network/routes_dynamic_used: - network/routes_dynamic_used_ratio: - network/routes_static_used: - network/subnets_available: - network/subnets_used: - network/subnets_used_ratio: - peering_group/forwarding_rules_l4_available: - peering_group/forwarding_rules_l4_used: - peering_group/forwarding_rules_l4_used_ratio: - peering_group/forwarding_rules_l7_available: - peering_group/forwarding_rules_l7_used: - peering_group/forwarding_rules_l7_used_ratio: - peering_group/instances_available: - peering_group/instances_used: - peering_group/instances_used_ratio: - peering_group/routes_dynamic_available: - peering_group/routes_dynamic_used: - peering_group/routes_dynamic_used_ratio: - peering_group/routes_static_available: - peering_group/routes_static_used: - peering_group/routes_static_used_ratio: - project/firewall_rules_available: - project/firewall_rules_used: - project/firewall_rules_used_ratio: - project/routes_static_available: - project/routes_static_used: - project/routes_static_used_ratio: - subnetwork/addresses_available: - subnetwork/addresses_used: - subnetwork/addresses_used_ratio: + firewall_policies/tuples_available: Firewall tuples limit per policy + firewall_policies/tuples_used: Firewall tuples used per policy + firewall_policies/tuples_used_ratio: Firewall tuples used ratio per policy + network/firewall_rules/used: Firewall rules used used per network + network/forwarding_rules_l7_available: L7 forwarding rules limit per network + network/forwarding_rules_l7_used: L7 forwarding rules used per network + network/forwarding_rules_l7_used_ratio: L7 forwarding rules used ratio per network + network/instances_available: Instance limit per network + network/instances_used: Instance used per network + network/instances_used_ratio: Instance used ratio per network + network/peerings_active_available: Active peering limit per network + network/peerings_active_used: Active peering used per network + network/peerings_active_used_ratio: Active peering used ratio per network + network/peerings_total_available: Total peering limit per network + network/peerings_total_used: Total peering used per network + network/peerings_total_used_ratio: Total peering used ratio per network + network/routes_dynamic_available: Dynamic routes limit per network + network/routes_dynamic_used: Dynamic routes used per network + network/routes_dynamic_used_ratio: Dynamic routes used ratio per network + network/routes_static_used: Static routes used per network + network/subnets_available: Subnet limit per network + network/subnets_used: Subnet used per network + network/subnets_used_ratio: Subnet used ratio per network + peering_group/forwarding_rules_l4_available: L4 forwarding rules limit per peering group + peering_group/forwarding_rules_l4_used: L4 forwarding rules used per peering group + peering_group/forwarding_rules_l4_used_ratio: L4 forwarding rules used ratio per peering group + peering_group/forwarding_rules_l7_available: L7 forwarding rules limit per peering group + peering_group/forwarding_rules_l7_used: L7 forwarding rules used per peering group + peering_group/forwarding_rules_l7_used_ratio: L7 forwarding rules used ratio per peering group + peering_group/instances_available: Instance limit per peering group + peering_group/instances_used: Instances used per peering group + peering_group/instances_used_ratio: Instances used ratio per peering group + peering_group/routes_dynamic_available: Dynamic routes limit per peering group + peering_group/routes_dynamic_used: Dynamic routes used per peering group + peering_group/routes_dynamic_used_ratio: Dynamic routes used ratio per peering group + peering_group/routes_static_available: Static routes limit per peering group + peering_group/routes_static_used: Static routes used per peering group + peering_group/routes_static_used_ratio: Static routes used ratio per peering group + project/firewall_rules_available: Firewall rules limit per project + project/firewall_rules_used: Firewall rules used per project + project/firewall_rules_used_ratio: Firewall rules used ratio per project + project/routes_static_available: Static rotues limit per project + project/routes_static_used: Static rotues used per project + project/routes_static_used_ratio: Static rotues used ratio per project + subnetwork/addresses_available: Address limit per project + subnetwork/addresses_used: Addresses used per project + subnetwork/addresses_used_ratio: Addresses used ratio per project diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py index 4534ba1984..e2dc0f99c5 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py @@ -14,8 +14,15 @@ import logging -from . import TimeSeries, register_timeseries +from . import MetricDescriptor, TimeSeries, register_timeseries +DESCRIPTOR_PREFIX = 'firewall_policies' +DESCRIPTOR_ATTRS = { + 'tuples_used': 'Firewall tuples used per policy', + 'tuples_available': 'Firewall tuples limit per policy', + 'tuples_used_ratio': 'Firewall tuples used ratio per policy' +} +DESCRIPTOR_LABELS = ('parent', 'name') LOGGER = logging.getLogger('net-dash.timeseries.firewall-policies') TUPLE_LIMIT = 2000 @@ -24,6 +31,9 @@ def timeseries(resources): 'Derive network timeseries for firewall policies.' LOGGER.info('timeseries') + for dtype, name in DESCRIPTOR_ATTRS.items(): + yield MetricDescriptor(f'{DESCRIPTOR_PREFIX}/{dtype}', name, + DESCRIPTOR_LABELS, dtype.endswith('ratio')) for v in resources['firewall_policies'].values(): tuples = int(v['num_tuples']) labels = {'parent': v['parent'], 'name': v['name']} diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index 7c967ef759..db250f8412 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -15,8 +15,13 @@ import itertools import logging -from . import TimeSeries, register_timeseries +from . import MetricDescriptor, TimeSeries, register_timeseries +DESCRIPTOR_ATTRS = { + 'firewall_rules_used': 'Firewall rules used per {}', + 'firewall_rules_available': 'Firewall rules limit per {}', + 'firewall_rules_used_ratio': 'Firewall rules used ratio per {}', +} LOGGER = logging.getLogger('net-dash.timeseries.firewall-rules') @@ -24,13 +29,18 @@ def timeseries(resources): 'Derive and yield network and project timeseries for firewall rules.' LOGGER.info('timeseries') + for prefix in ('network', 'project'): + for dtype, name in DESCRIPTOR_ATTRS.items(): + labels = ('project',) if prefix == 'project' else ('project', 'name') + yield MetricDescriptor(f'{prefix}/{dtype}', name.format(prefix), labels, + name.endswith('ratio')) grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['network']) for network_id, rules in grouped: count = len(list(rules)) labels = { - 'network_name': resources['networks'][network_id]['name'], - 'project_id': resources['networks'][network_id]['project_id'] + 'name': resources['networks'][network_id]['name'], + 'project': resources['networks'][network_id]['project_id'] } yield TimeSeries('network/firewall_rules/used', count, labels) grouped = itertools.groupby(resources['firewall_rules'].values(), @@ -38,7 +48,7 @@ def timeseries(resources): for project_id, rules in grouped: count = len(list(rules)) limit = int(resources['quota'][project_id]['global']['FIREWALLS']) - labels = {'project_id': project_id} + labels = {'project': project_id} yield TimeSeries('project/firewall_rules_used', count, labels) yield TimeSeries('project/firewall_rules_available', limit, labels) yield TimeSeries('project/firewall_rules_used_ratio', count / limit, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index a789c29c6e..1bffa0c623 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -17,8 +17,30 @@ import logging import operator -from . import TimeSeries, register_timeseries +from . import MetricDescriptor, TimeSeries, register_timeseries +DESCRIPTOR_PREFIX = 'network' +DESCRIPTOR_ATTRS = { + 'forwarding_rules_l4_available': 'L4 fwd rules limit per network', + 'forwarding_rules_l4_used': 'L4 fwd rules used per network', + 'forwarding_rules_l4_used_ratio': 'L4 fwd rules used ratio per network', + 'forwarding_rules_l7_available': 'L7 fwd rules limit per network', + 'forwarding_rules_l7_used': 'L7 fwd rules used per network', + 'forwarding_rules_l7_used_ratio': 'L7 fwd rules used ratio per network', + 'instances_available': 'Instance limit per network', + 'instances_used': 'Instance used per network', + 'instances_used_ratio': 'Instance used ratio per network', + 'peerings_active_available': 'Active peering limit per network', + 'peerings_active_used': 'Active peering used per network', + 'peerings_active_used_ratio': 'Active peering used ratio per network', + 'peerings_total_available': 'Total peering limit per network', + 'peerings_total_used': 'Total peering used per network', + 'peerings_total_used_ratio': 'Total peering used ratio per network', + 'subnets_available': 'Subnet limit per network', + 'subnets_used': 'Subnet used per network', + 'subnets_used_ratio': 'Subnet used ratio per network' +} +DESCRIPTOR_LABELS = ('project', 'network') LIMITS = { 'INSTANCES_PER_NETWORK_GLOBAL': 15000, 'INTERNAL_FORWARDING_RULES_PER_NETWORK': 500, @@ -101,5 +123,8 @@ def _subnet_ranges(resources): def timeseries(resources): 'Yield timeseries.' LOGGER.info('timeseries') + for dtype, name in DESCRIPTOR_ATTRS.items(): + yield MetricDescriptor(f'{DESCRIPTOR_PREFIX}/{dtype}', name, + DESCRIPTOR_LABELS, dtype.endswith('ratio')) return itertools.chain(_forwarding_rules(resources), _instances(resources), _peerings(resources), _subnet_ranges(resources)) From 7e49eda07dc1f5e1974231b965a5e3374df56060 Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 24 Nov 2022 11:55:07 +0100 Subject: [PATCH 38/82] metric descriptors --- .../cf/plugins/descriptors.yaml | 49 ------------------- .../cf/plugins/series-networks.py | 6 +-- .../cf/plugins/series-peering-groups.py | 39 ++++++++++++++- .../cf/plugins/series-routes.py | 22 ++++++++- .../cf/plugins/series-subnets.py | 11 ++++- 5 files changed, 70 insertions(+), 57 deletions(-) delete mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml b/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml deleted file mode 100644 index c03a37bf91..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/descriptors.yaml +++ /dev/null @@ -1,49 +0,0 @@ -prefix: net-dash/ -descriptors: - firewall_policies/tuples_available: Firewall tuples limit per policy - firewall_policies/tuples_used: Firewall tuples used per policy - firewall_policies/tuples_used_ratio: Firewall tuples used ratio per policy - network/firewall_rules/used: Firewall rules used used per network - network/forwarding_rules_l7_available: L7 forwarding rules limit per network - network/forwarding_rules_l7_used: L7 forwarding rules used per network - network/forwarding_rules_l7_used_ratio: L7 forwarding rules used ratio per network - network/instances_available: Instance limit per network - network/instances_used: Instance used per network - network/instances_used_ratio: Instance used ratio per network - network/peerings_active_available: Active peering limit per network - network/peerings_active_used: Active peering used per network - network/peerings_active_used_ratio: Active peering used ratio per network - network/peerings_total_available: Total peering limit per network - network/peerings_total_used: Total peering used per network - network/peerings_total_used_ratio: Total peering used ratio per network - network/routes_dynamic_available: Dynamic routes limit per network - network/routes_dynamic_used: Dynamic routes used per network - network/routes_dynamic_used_ratio: Dynamic routes used ratio per network - network/routes_static_used: Static routes used per network - network/subnets_available: Subnet limit per network - network/subnets_used: Subnet used per network - network/subnets_used_ratio: Subnet used ratio per network - peering_group/forwarding_rules_l4_available: L4 forwarding rules limit per peering group - peering_group/forwarding_rules_l4_used: L4 forwarding rules used per peering group - peering_group/forwarding_rules_l4_used_ratio: L4 forwarding rules used ratio per peering group - peering_group/forwarding_rules_l7_available: L7 forwarding rules limit per peering group - peering_group/forwarding_rules_l7_used: L7 forwarding rules used per peering group - peering_group/forwarding_rules_l7_used_ratio: L7 forwarding rules used ratio per peering group - peering_group/instances_available: Instance limit per peering group - peering_group/instances_used: Instances used per peering group - peering_group/instances_used_ratio: Instances used ratio per peering group - peering_group/routes_dynamic_available: Dynamic routes limit per peering group - peering_group/routes_dynamic_used: Dynamic routes used per peering group - peering_group/routes_dynamic_used_ratio: Dynamic routes used ratio per peering group - peering_group/routes_static_available: Static routes limit per peering group - peering_group/routes_static_used: Static routes used per peering group - peering_group/routes_static_used_ratio: Static routes used ratio per peering group - project/firewall_rules_available: Firewall rules limit per project - project/firewall_rules_used: Firewall rules used per project - project/firewall_rules_used_ratio: Firewall rules used ratio per project - project/routes_static_available: Static rotues limit per project - project/routes_static_used: Static rotues used per project - project/routes_static_used_ratio: Static rotues used ratio per project - subnetwork/addresses_available: Address limit per project - subnetwork/addresses_used: Addresses used per project - subnetwork/addresses_used_ratio: Addresses used ratio per project diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index 1bffa0c623..af299e7a4f 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -19,7 +19,6 @@ from . import MetricDescriptor, TimeSeries, register_timeseries -DESCRIPTOR_PREFIX = 'network' DESCRIPTOR_ATTRS = { 'forwarding_rules_l4_available': 'L4 fwd rules limit per network', 'forwarding_rules_l4_used': 'L4 fwd rules used per network', @@ -40,7 +39,6 @@ 'subnets_used': 'Subnet used per network', 'subnets_used_ratio': 'Subnet used ratio per network' } -DESCRIPTOR_LABELS = ('project', 'network') LIMITS = { 'INSTANCES_PER_NETWORK_GLOBAL': 15000, 'INTERNAL_FORWARDING_RULES_PER_NETWORK': 500, @@ -124,7 +122,7 @@ def timeseries(resources): 'Yield timeseries.' LOGGER.info('timeseries') for dtype, name in DESCRIPTOR_ATTRS.items(): - yield MetricDescriptor(f'{DESCRIPTOR_PREFIX}/{dtype}', name, - DESCRIPTOR_LABELS, dtype.endswith('ratio')) + yield MetricDescriptor(f'network/{dtype}', name, ('project', 'network'), + dtype.endswith('ratio')) return itertools.chain(_forwarding_rules(resources), _instances(resources), _peerings(resources), _subnet_ranges(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py index 758ecbfb42..4f2f2c2311 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py @@ -15,8 +15,40 @@ import itertools import logging -from . import TimeSeries, register_timeseries - +from . import MetricDescriptor, TimeSeries, register_timeseries + +DESCRIPTOR_ATTRS = { + 'forwarding_rules_l4_available': + 'L4 fwd rules limit per peering group', + 'forwarding_rules_l4_used': + 'L4 fwd rules used per peering group', + 'forwarding_rules_l4_used_ratio': + 'L4 fwd rules used ratio per peering group', + 'forwarding_rules_l7_available': + 'L7 fwd rules limit per peering group', + 'forwarding_rules_l7_used': + 'L7 fwd rules used per peering group', + 'forwarding_rules_l7_used_ratio': + 'L7 fwd rules used ratio per peering group', + 'instances_available': + 'Instance limit per peering group', + 'instances_used': + 'Instance used per peering group', + 'instances_used_ratio': + 'Instance used ratio per peering group', + 'routes_dynamic_available': + 'Dynamic route limit per peering group', + 'routes_dynamic_used': + 'Dynamic route used per peering group', + 'routes_dynamic_used_ratio': + 'Dynamic route used ratio per peering group', + 'routes_static_available': + 'Static route limit per peering group', + 'routes_static_used': + 'Static route used per peering group', + 'routes_static_used_ratio': + 'Static route used ratio per peering group', +} LIMITS = { 'forwarding_rules_l4': { 'pg': ('INTERNAL_FORWARDING_RULES_PER_PEERING_GROUP', 500), @@ -128,5 +160,8 @@ def _network_timeseries(resources, network): def timeseries(resources): 'Yield timeseries.' LOGGER.info('timeseries') + for dtype, name in DESCRIPTOR_ATTRS.items(): + yield MetricDescriptor(f'peering_group/{dtype}', name, + ('project', 'network'), dtype.endswith('ratio')) return itertools.chain(*(_network_timeseries(resources, n) for n in resources['networks'].values())) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py index 3502887fb8..8a97222323 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py @@ -15,8 +15,24 @@ import itertools import logging -from . import TimeSeries, register_timeseries +from . import MetricDescriptor, TimeSeries, register_timeseries +DESCRIPTOR_ATTRS = { + 'network/routes_dynamic_used': + 'Dynamic routes limit per network', + 'network/routes_dynamic_available': + 'Dynamic routes used per network', + 'network/routes_dynamic_used_ratio': + 'Dynamic routes used ratio per network', + 'network/routes_static_used': + 'Static routes limit per network', + 'project/routes_dynamic_used': + 'Dynamic routes limit per project', + 'project/routes_dynamic_available': + 'Dynamic routes used per project', + 'project/routes_dynamic_used_ratio': + 'Dynamic routes used ratio per project' +} LIMITS = {'ROUTES': 250, 'ROUTES_DYNAMIC': 100} LOGGER = logging.getLogger('net-dash.timeseries.routes') @@ -59,4 +75,8 @@ def _static(resources): def timeseries(resources): 'Yield timeseries.' LOGGER.info('timeseries') + for dtype, name in DESCRIPTOR_ATTRS.items(): + labels = ('project') if dtype.startswith('project') else ('project', + 'network') + yield MetricDescriptor(dtype, name, labels, dtype.endswith('ratio')) return itertools.chain(_static(resources), _dynamic(resources)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index 41c896a107..c1313aba6a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -17,8 +17,13 @@ import itertools import logging -from . import TimeSeries, register_timeseries +from . import MetricDescriptor, TimeSeries, register_timeseries +DESCRIPTOR_ATTRS = { + 'addresses_available': 'Address limit per subnet', + 'addresses_used': 'Addresses used per subnet', + 'addresses_used_ratio': 'Addresses used ratio per subnet' +} LOGGER = logging.getLogger('net-dash.timeseries.subnets') @@ -61,6 +66,10 @@ def _subnet_instances(resources): def timeseries(resources): 'Derive and yield subnetwork timeseries for address utilization.' LOGGER.info('timeseries') + for dtype, name in DESCRIPTOR_ATTRS.items(): + yield MetricDescriptor(f'subnetwork/{dtype}', name, + ('project', 'network', 'subnetwork'), + dtype.endswith('ratio')) subnet_nets = { k: ipaddress.ip_network(v['cidr_range']) for k, v in resources['subnetworks'].items() From b9bfb47a189adb04fccd4ad1dcb415628ecef79e Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 24 Nov 2022 12:59:38 +0100 Subject: [PATCH 39/82] fixes --- .../network-dashboard/cf/out.json | 86 ++++++++++++++----- .../cf/plugins/discover-compute-quota.py | 2 +- ...rics.py => discover-metric-descriptors.py} | 2 +- 3 files changed, 68 insertions(+), 22 deletions(-) rename blueprints/cloud-operations/network-dashboard/cf/plugins/{discover-metrics.py => discover-metric-descriptors.py} (98%) diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 740a0fc4de..6301cbb331 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -9,20 +9,7 @@ "config:projects": [ "tf-playground-simple" ], - "config:custom_quota": { - "projects": { - "tf-playground-svpc-net": { - "global": { - "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 - } - } - }, - "networks": { - "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "PEERINGS_PER_NETWORK": 40 - } - } - }, + "config:custom_quota": {}, "addresses": { "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello": { "id": "2569728380045293941", @@ -411,6 +398,14 @@ "project_id": "tf-playground-svpc-net", "project_number": "1079408472053", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" + }, + "projects/tf-playground-svpc-net/global/firewalls/test-ingress-echo": { + "id": "1793500710165447459", + "name": "test-ingress-echo", + "self_link": "projects/tf-playground-svpc-net/global/firewalls/test-ingress-echo", + "project_id": "tf-playground-svpc-net", + "project_number": "1079408472053", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" } }, "forwarding_rules": { @@ -427,6 +422,19 @@ "region": "europe-west8", "subnetwork": null }, + "projects/tf-playground-svpc-gce/regions/europe-west8/forwardingRules/test": { + "id": "5258452901764956944", + "name": "test", + "self_link": "projects/tf-playground-svpc-gce/regions/europe-west8/forwardingRules/test", + "project_id": "tf-playground-svpc-gce", + "project_number": "783093469136", + "address": "10.24.32.33", + "load_balancing_scheme": "INTERNAL", + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "psc_accepted": false, + "region": "europe-west8", + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + }, "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/bastion-wg": { "id": "4724174290265929790", "name": "bastion-wg", @@ -469,6 +477,48 @@ } ] }, + "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-0": { + "id": "8688603441516413730", + "name": "test-0", + "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-0", + "project_id": "tf-playground-svpc-gce", + "project_number": "783093469136", + "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", + "networks": [ + { + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + } + ] + }, + "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-1": { + "id": "6078779822666056482", + "name": "test-1", + "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-1", + "project_id": "tf-playground-svpc-gce", + "project_number": "783093469136", + "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", + "networks": [ + { + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + } + ] + }, + "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-failover": { + "id": "104671418499892002", + "name": "test-failover", + "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-failover", + "project_id": "tf-playground-svpc-gce", + "project_number": "783093469136", + "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", + "networks": [ + { + "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", + "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" + } + ] + }, "projects/tf-playground-svpc-gce/zones/europe-west8-c/instances/nginx-ew8-c": { "id": "4082958655368747304", "name": "nginx-ew8-c", @@ -2591,8 +2641,7 @@ "URL_MAPS": 30, "VPN_GATEWAYS": 15, "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000, - "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 + "XPN_SERVICE_PROJECTS": 1000 } }, "tf-playground-svpc-gke": { @@ -2639,9 +2688,6 @@ "VPN_TUNNELS": 30, "XPN_SERVICE_PROJECTS": 1000 } - }, - "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "PEERINGS_PER_NETWORK": 40 } }, "routes_dynamic": { @@ -2670,5 +2716,5 @@ "projects/tf-playground-svpc-net/global/networks/shared-vpc" ] }, - "metrics": {} + "metric-descriptors": {} } \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 093cd3d520..3d624dc7d1 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -27,7 +27,7 @@ def _handle_discovery(resources, response): LOGGER.info('discovery handle request') content_type = response.headers['content-type'] - per_project_quota = resources['config:custom_quota'].get('projects') + per_project_quota = resources['config:custom_quota'].get('projects', {}) for part in dirty_mp_response(content_type, response.content): kind = part.get('kind') quota = { diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py similarity index 98% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py index 3ee542792d..75c319361d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py @@ -19,7 +19,7 @@ from .utils import parse_page_token LOGGER = logging.getLogger('net-dash.discovery.metrics') -NAME = 'metrics' +NAME = 'metric-descriptors' URL = ( 'https://content-monitoring.googleapis.com/v3/projects' From bf82c85abad205a4e2fc0d7ea3e3574c9aa6d16a Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 24 Nov 2022 13:53:46 +0100 Subject: [PATCH 40/82] wip --- .../network-dashboard/cf/main.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index dc4a554531..b8af5349b1 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -56,6 +56,8 @@ def do_discovery(resources): resources[result.type][result.id][result.key] = result.data else: resources[result.type][result.id] = result.data + LOGGER.info('discovery end {}'.format( + {k: len(v) for k, v in resources.items() if not isinstance(v, str)})) def do_init(resources, organization, op_project, folders=None, projects=None, @@ -76,9 +78,9 @@ def do_metric_descriptors(timeseries, op_project): pass -def do_timeseries(resources, descriptors, timeseries, debug_plugin=None): - 'Create timeseries.' - LOGGER.info(f'timeseries start (debug plugin: {debug_plugin})') +def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): + 'Calculate timeseries.' + LOGGER.info(f'timeseries calc start (debug plugin: {debug_plugin})') for plugin in plugins.get_timeseries_plugins(): if debug_plugin and plugin.name != debug_plugin: LOGGER.info(f'skipping {plugin.name}') @@ -90,6 +92,13 @@ def do_timeseries(resources, descriptors, timeseries, debug_plugin=None): descriptors[result.type] = result elif isinstance(result, plugins.TimeSeries): timeseries.append(result) + LOGGER.info('timeseries calc end (descriptors: {} timeseries: {})'.format( + len(descriptors), len(timeseries))) + + +def do_timeseries_post(descriptors, timeseries): + 'Post timeseries.' + LOGGER.info('timeseries post start') def fetch(request): @@ -146,16 +155,10 @@ def main(organization=None, op_project=None, project=None, folder=None, raise SystemExit(f'Error decoding custom quota file: {e.args[0]}') do_init(resources, organization, op_project, folder, project, custom_quota) do_discovery(resources) - do_timeseries(resources, descriptors, timeseries, debug_plugin) - LOGGER.info( - {k: len(v) for k, v in resources.items() if not isinstance(v, str)}) - LOGGER.info(f'{len(timeseries)} timeseries') - - if dump_file: - json.dump(resources, dump_file, indent=2) - - import icecream - icecream.ic(descriptors) + if dump_file: + json.dump(resources, dump_file, indent=2) + do_timeseries_calc(resources, descriptors, timeseries, debug_plugin) + do_timeseries_post(descriptors, timeseries) if __name__ == '__main__': From 1ba7ac8979d6f119fd5e2b049dac43fc2a800fe8 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 14:42:39 +0100 Subject: [PATCH 41/82] Use partial for all cf init functions --- .../network-dashboard/cf/plugins/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index d4f444e0b3..4e066dc776 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -59,9 +59,9 @@ def outer(func): return outer -get_discovery_plugins = lambda: iter(_PLUGINS_DISCOVERY) -get_init_plugins = lambda: iter(_PLUGINS_INIT) -get_timeseries_plugins = lambda: iter(_PLUGINS_TIMESERIES) +get_discovery_plugins = functools.partial(iter, _PLUGINS_DISCOVERY) +get_init_plugins = functools.partial(iter, _PLUGINS_INIT) +get_timeseries_plugins = functools.partial(iter, _PLUGINS_TIMESERIES) register_discovery = functools.partial(_register_plugin, _PLUGINS_DISCOVERY) register_init = functools.partial(_register_plugin, _PLUGINS_INIT) register_timeseries = functools.partial(_register_plugin, _PLUGINS_TIMESERIES) From a9b4806c04c72690227be2a9323e0ec7151fc94f Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 15:03:44 +0100 Subject: [PATCH 42/82] Add requirements.txt --- .../cloud-operations/network-dashboard/cf/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/requirements.txt diff --git a/blueprints/cloud-operations/network-dashboard/cf/requirements.txt b/blueprints/cloud-operations/network-dashboard/cf/requirements.txt new file mode 100644 index 0000000000..3ca529bc35 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/requirements.txt @@ -0,0 +1,4 @@ +click==8.1.3 +google-auth==2.14.1 +PyYAML==6.0 +requests==2.28.1 From 045112d6f876a5c03f10b84dc8aa0120f2df7e9e Mon Sep 17 00:00:00 2001 From: Ludo Date: Thu, 24 Nov 2022 15:06:54 +0100 Subject: [PATCH 43/82] fix org key mismatch --- .../network-dashboard/cf/plugins/discover-cai-compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index f27343afbc..34ad5a136d 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -186,7 +186,7 @@ def _get_parent(parent, resources): if parent_type == 'folders': if parent_id in resources['config:folders']: return {'parent': f'{parent_type}/{parent_id}'} - if resources['organization'] == int(parent_id): + if resources['config:organization'] == int(parent_id): return {'parent': f'{parent_type}/{parent_id}'} From a04bbf5d2edebbed101154bad296d9caebed8eb8 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 15:40:11 +0100 Subject: [PATCH 44/82] Fix folder short cli name --- blueprints/cloud-operations/network-dashboard/cf/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index b8af5349b1..b972e1135b 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -127,7 +127,7 @@ def fetch(request): help='GCP monitoring project where metrics will be stored') @click.option('--project', '-p', type=str, multiple=True, help='GCP project id, can be specified multiple times') -@click.option('--folder', '-p', type=int, multiple=True, +@click.option('--folder', '-f', type=int, multiple=True, help='GCP folder id, can be specified multiple times') @click.option('--custom-quota-file', type=click.File('r'), help='Custom quota file in yaml format.') From 897e873e49c22778357f11a2530a8fa39be54211 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 15:41:09 +0100 Subject: [PATCH 45/82] Fix instance_networks when iterable is empty --- .../network-dashboard/cf/plugins/series-networks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index af299e7a4f..986d6e9ac2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -85,8 +85,8 @@ def _forwarding_rules(resources): def _instances(resources): 'Derive network timeseries for instance utilization.' - instance_networks = functools.reduce( - operator.add, [i['networks'] for i in resources['instances'].values()]) + instance_networks = itertools.chain.from_iterable( + i['networks'] for i in resources['instances'].values()) grouped = itertools.groupby(instance_networks, lambda i: i['network']) return _group_timeseries('instances', resources, grouped, 'INSTANCES_PER_NETWORK_GLOBAL') From e5e1c662c584aeae8e527084547a5e575beffd14 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 16:22:28 +0100 Subject: [PATCH 46/82] more readability and fixing some strings --- blueprints/cloud-operations/network-dashboard/cf/main.py | 8 ++++---- .../network-dashboard/cf/plugins/__init__.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index b972e1135b..050853c1dc 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -122,13 +122,13 @@ def fetch(request): @click.command() @click.option('--organization', '-o', required=True, type=int, - help='GCP organization id') + help='GCP organization id.') @click.option('--op-project', '-op', required=True, type=str, - help='GCP monitoring project where metrics will be stored') + help='GCP monitoring project where metrics will be stored.') @click.option('--project', '-p', type=str, multiple=True, - help='GCP project id, can be specified multiple times') + help='GCP project id, can be specified multiple times.') @click.option('--folder', '-f', type=int, multiple=True, - help='GCP folder id, can be specified multiple times') + help='GCP folder id, can be specified multiple times.') @click.option('--custom-quota-file', type=click.File('r'), help='Custom quota file in yaml format.') @click.option('--dump-file', type=click.File('w'), diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 4e066dc776..b1244392ea 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -71,4 +71,4 @@ def outer(func): for mod_info in pkgutil.iter_modules([_plugins_path], 'plugins.'): importlib.import_module(mod_info.name) -_PLUGINS_DISCOVERY.sort(key=lambda i: i[2:-1]) +_PLUGINS_DISCOVERY.sort(key=lambda i: i.level) From e64c3bb6493bc87b20596ca24dc1b9950120c7a0 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 17:09:05 +0100 Subject: [PATCH 47/82] replace() -> removeprefix and remove unneeded quoting --- .../cf/plugins/discover-cai-compute.py | 10 +++------- .../cf/plugins/discover-cai-projects.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 34ad5a136d..4335da3a3a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -13,7 +13,6 @@ # limitations under the License. import logging -import urllib.parse from . import HTTPRequest, Level, Resource, register_init, register_discovery from .utils import parse_cai_results @@ -151,9 +150,7 @@ def _handle_routers(resource, data): def _handle_routes(resource, data): 'Handle route type resource data.' - hop = [ - a.replace('nextHop', '').lower() for a in data if a.startswith('nextHop') - ] + hop = [a.removeprefix('nextHop').lower() for a in data] return {'next_hop_type': hop[0], 'network': _self_link(data['network'])} @@ -173,7 +170,7 @@ def _handle_subnetworks(resource, data): def _self_link(s): 'Remove initial part from self links.' - return s.replace('https://www.googleapis.com/compute/v1/', '') + return s.removeprefix('https://www.googleapis.com/compute/v1/') def _get_parent(parent, resources): @@ -194,8 +191,7 @@ def _url(resources): 'Return discovery URL' organization = resources['config:organization'] asset_types = '&'.join( - 'assetTypes=compute.googleapis.com/{}'.format(urllib.parse.quote(t)) - for t in TYPES.values()) + f'assetTypes=compute.googleapis.com/{t}' for t in TYPES.values()) return CAI_URL.format(organization=organization, asset_types=asset_types) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index 0730c369d6..ac2bed048e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -24,7 +24,7 @@ CAI_URL = ( 'https://content-cloudasset.googleapis.com/v1p1beta1' '/{}/resources:searchAll' - f'?assetTypes=cloudresourcemanager.googleapis.com%2FProject&pageSize=500') + f'?assetTypes=cloudresourcemanager.googleapis.com/Project&pageSize=500') def _handle_discovery(resources, response, data): From 7b70ee53ac71b4d7f5224836fc10cff40d091c68 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 17:48:02 +0100 Subject: [PATCH 48/82] setdefault in init()s --- .../network-dashboard/cf/plugins/discover-cai-compute.py | 3 +-- .../network-dashboard/cf/plugins/discover-cai-projects.py | 6 ++---- .../network-dashboard/cf/plugins/discover-compute-quota.py | 3 +-- .../cf/plugins/discover-compute-routerstatus.py | 3 +-- .../network-dashboard/cf/plugins/discover-group-networks.py | 3 +-- .../cf/plugins/discover-metric-descriptors.py | 3 +-- 6 files changed, 7 insertions(+), 14 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 4335da3a3a..1378d2b2f0 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -200,8 +200,7 @@ def init(resources): 'Prepare the shared datastructures for asset types managed here.' LOGGER.info('init') for name in TYPES: - if name not in resources: - resources[name] = {} + resources.setdefault(name, {}) @register_discovery(Level.PRIMARY, 10) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index ac2bed048e..8358421857 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -45,10 +45,8 @@ def _handle_discovery(resources, response, data): @register_init def init(resources): LOGGER.info('init') - if NAME not in resources: - resources[NAME] = {} - if 'project:numbers' not in resources: - resources['projects:number'] = {} + resources.setdefault(NAME, {}) + resources.setdefault('projects:number', {}) @register_discovery(Level.CORE, 0) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 3d624dc7d1..4c5fa3ca95 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -56,8 +56,7 @@ def _handle_discovery(resources, response): def init(resources): 'Create the quota key in the shared resource map.' LOGGER.info('init') - if NAME not in resources: - resources[NAME] = {} + resources.setdefault(NAME, {}) @register_discovery(Level.DERIVED, 0) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index cd185e004f..b40e05f0d6 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -58,8 +58,7 @@ def _handle_discovery(resources, response): @register_init def init(resources): LOGGER.info('init') - if NAME not in resources: - resources[NAME] = {} + resources.setdefault(NAME, {}) @register_discovery(Level.DERIVED) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py index d2e4b51021..1b5f12c89e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py @@ -24,8 +24,7 @@ @register_init def init(resources): LOGGER.info('init') - if NAME not in resources: - resources[NAME] = {} + resources.setdefault(NAME, {}) @register_discovery(Level.DERIVED) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py index 75c319361d..96d826c248 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py @@ -45,8 +45,7 @@ def _handle_discovery(resources, response, data): @register_init def init(resources): LOGGER.info('init') - if NAME not in resources: - resources[NAME] = {} + resources.setdefault(NAME, {}) @register_discovery(Level.CORE, 0) From 1854216ffeedb7cce4f072c7dbcf4f0e22feed44 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 18:09:42 +0100 Subject: [PATCH 49/82] Fix next hop type --- .../network-dashboard/cf/plugins/discover-cai-compute.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 1378d2b2f0..2192b44ce2 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -150,7 +150,9 @@ def _handle_routers(resource, data): def _handle_routes(resource, data): 'Handle route type resource data.' - hop = [a.removeprefix('nextHop').lower() for a in data] + hop = [ + a.removeprefix('nextHop').lower() for a in data if a.startswith('nextHop') + ] return {'next_hop_type': hop[0], 'network': _self_link(data['network'])} From 0d2ce943f46faee7dba3c4c8d64ea9a7cf8060c8 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 24 Nov 2022 18:09:50 +0100 Subject: [PATCH 50/82] Remove unneeded fstring --- .../network-dashboard/cf/plugins/discover-cai-projects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py index 8358421857..862f99e92e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py @@ -24,7 +24,7 @@ CAI_URL = ( 'https://content-cloudasset.googleapis.com/v1p1beta1' '/{}/resources:searchAll' - f'?assetTypes=cloudresourcemanager.googleapis.com/Project&pageSize=500') + '?assetTypes=cloudresourcemanager.googleapis.com/Project&pageSize=500') def _handle_discovery(resources, response, data): From f7a4b65e79aedc55c304826caa282ecf7ceb7fd0 Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 25 Nov 2022 08:38:57 +0100 Subject: [PATCH 51/82] create descriptors --- .../network-dashboard/cf/main.py | 31 ++++--- .../network-dashboard/cf/out.json | 93 +++++++++++++++---- .../cf/plugins/discover-metric-descriptors.py | 17 ++-- .../cf/plugins/monitoring.py | 53 +++++++++++ 4 files changed, 157 insertions(+), 37 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 050853c1dc..20ea4b681c 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -20,12 +20,14 @@ import click import google.auth import plugins +import plugins.monitoring import yaml from google.auth.transport.requests import AuthorizedSession HTTP = AuthorizedSession(google.auth.default()[0]) LOGGER = logging.getLogger('net-dash') +MONITORING_ROOT = 'netmon/' Result = collections.namedtuple('Result', 'phase resource data') @@ -69,17 +71,13 @@ def do_init(resources, organization, op_project, folders=None, projects=None, resources['config:folders'] = [str(f) for f in folders or []] resources['config:projects'] = projects or [] resources['config:custom_quota'] = custom_quota or {} + resources['config:monitoring_root'] = MONITORING_ROOT for plugin in plugins.get_init_plugins(): plugin.func(resources) -def do_metric_descriptors(timeseries, op_project): - 'Create missing descriptors for custom metrics in timeseries.' - pass - - def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): - 'Calculate timeseries.' + 'Calculate descriptors and timeseries.' LOGGER.info(f'timeseries calc start (debug plugin: {debug_plugin})') for plugin in plugins.get_timeseries_plugins(): if debug_plugin and plugin.name != debug_plugin: @@ -89,16 +87,25 @@ def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): if not result: continue if isinstance(result, plugins.MetricDescriptor): - descriptors[result.type] = result + descriptors.append(result) elif isinstance(result, plugins.TimeSeries): timeseries.append(result) LOGGER.info('timeseries calc end (descriptors: {} timeseries: {})'.format( len(descriptors), len(timeseries))) -def do_timeseries_post(descriptors, timeseries): +def do_timeseries_descriptors(project_id, existing, computed): + 'Post timeseries descriptors.' + LOGGER.info('timeseries descriptors start') + urls = plugins.monitoring.create_descriptors(project_id, MONITORING_ROOT, + existing, computed) + for url in urls: + fetch(url) + + +def do_timeseries(project_id, timeseries): 'Post timeseries.' - LOGGER.info('timeseries post start') + LOGGER.info('timeseries start') def fetch(request): @@ -141,7 +148,7 @@ def main(organization=None, op_project=None, project=None, folder=None, custom_quota_file=None, dump_file=None, load_file=None, debug_plugin=None): logging.basicConfig(level=logging.INFO) - descriptors = {} + descriptors = [] timeseries = [] if load_file: resources = json.load(load_file) @@ -158,7 +165,9 @@ def main(organization=None, op_project=None, project=None, folder=None, if dump_file: json.dump(resources, dump_file, indent=2) do_timeseries_calc(resources, descriptors, timeseries, debug_plugin) - do_timeseries_post(descriptors, timeseries) + do_timeseries_descriptors(op_project, resources['metric-descriptors'], + descriptors) + do_timeseries(op_project, timeseries) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 6301cbb331..36f327065c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -1,6 +1,6 @@ { "config:organization": "436789450919", - "config:monitoring_project": "google.com:ludo-sce-test", + "config:monitoring_project": "tf-playground-svpc-net", "config:folders": [ "821058723541", "949988871993", @@ -9,7 +9,21 @@ "config:projects": [ "tf-playground-simple" ], - "config:custom_quota": {}, + "config:custom_quota": { + "projects": { + "tf-playground-svpc-net": { + "global": { + "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 + } + } + }, + "networks": { + "projects/tf-playground-svpc-net/global/networks/shared-vpc": { + "PEERINGS_PER_NETWORK": 40 + } + } + }, + "config:monitoring_root": "netmon/", "addresses": { "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello": { "id": "2569728380045293941", @@ -505,20 +519,6 @@ } ] }, - "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-failover": { - "id": "104671418499892002", - "name": "test-failover", - "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-failover", - "project_id": "tf-playground-svpc-gce", - "project_number": "783093469136", - "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", - "networks": [ - { - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - } - ] - }, "projects/tf-playground-svpc-gce/zones/europe-west8-c/instances/nginx-ew8-c": { "id": "4082958655368747304", "name": "nginx-ew8-c", @@ -2641,7 +2641,8 @@ "URL_MAPS": 30, "VPN_GATEWAYS": 15, "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 + "XPN_SERVICE_PROJECTS": 1000, + "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 } }, "tf-playground-svpc-gke": { @@ -2688,6 +2689,9 @@ "VPN_TUNNELS": 30, "XPN_SERVICE_PROJECTS": 1000 } + }, + "projects/tf-playground-svpc-net/global/networks/shared-vpc": { + "PEERINGS_PER_NETWORK": 40 } }, "routes_dynamic": { @@ -2716,5 +2720,58 @@ "projects/tf-playground-svpc-net/global/networks/shared-vpc" ] }, - "metric-descriptors": {} + "metric-descriptors": { + "custom.googleapis.com/netmon/firewall_policies/tuples_available": {}, + "custom.googleapis.com/netmon/firewall_policies/tuples_used": {}, + "custom.googleapis.com/netmon/firewall_policies/tuples_used_ratio": {}, + "custom.googleapis.com/netmon/network/firewall_rules_available": {}, + "custom.googleapis.com/netmon/network/firewall_rules_used": {}, + "custom.googleapis.com/netmon/network/firewall_rules_used_ratio": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l4_available": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l4_used": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l7_available": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l7_used": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l7_used_ratio": {}, + "custom.googleapis.com/netmon/network/instances_available": {}, + "custom.googleapis.com/netmon/network/instances_used": {}, + "custom.googleapis.com/netmon/network/instances_used_ratio": {}, + "custom.googleapis.com/netmon/network/peerings_active_available": {}, + "custom.googleapis.com/netmon/network/peerings_active_used": {}, + "custom.googleapis.com/netmon/network/peerings_active_used_ratio": {}, + "custom.googleapis.com/netmon/network/peerings_total_available": {}, + "custom.googleapis.com/netmon/network/peerings_total_used": {}, + "custom.googleapis.com/netmon/network/peerings_total_used_ratio": {}, + "custom.googleapis.com/netmon/network/routes_dynamic_available": {}, + "custom.googleapis.com/netmon/network/routes_dynamic_used": {}, + "custom.googleapis.com/netmon/network/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/network/routes_static_used": {}, + "custom.googleapis.com/netmon/network/subnets_available": {}, + "custom.googleapis.com/netmon/network/subnets_used": {}, + "custom.googleapis.com/netmon/network/subnets_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_available": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_available": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/instances_available": {}, + "custom.googleapis.com/netmon/peering_group/instances_used": {}, + "custom.googleapis.com/netmon/peering_group/instances_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/routes_dynamic_available": {}, + "custom.googleapis.com/netmon/peering_group/routes_dynamic_used": {}, + "custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/routes_static_available": {}, + "custom.googleapis.com/netmon/peering_group/routes_static_used": {}, + "custom.googleapis.com/netmon/peering_group/routes_static_used_ratio": {}, + "custom.googleapis.com/netmon/project/firewall_rules_available": {}, + "custom.googleapis.com/netmon/project/firewall_rules_used": {}, + "custom.googleapis.com/netmon/project/firewall_rules_used_ratio": {}, + "custom.googleapis.com/netmon/project/routes_dynamic_available": {}, + "custom.googleapis.com/netmon/project/routes_dynamic_used": {}, + "custom.googleapis.com/netmon/project/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/subnetwork/addresses_available": {}, + "custom.googleapis.com/netmon/subnetwork/addresses_used": {}, + "custom.googleapis.com/netmon/subnetwork/addresses_used_ratio": {} + } } \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py index 96d826c248..509519bda7 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py @@ -21,11 +21,10 @@ LOGGER = logging.getLogger('net-dash.discovery.metrics') NAME = 'metric-descriptors' -URL = ( - 'https://content-monitoring.googleapis.com/v3/projects' - '/{}/metricDescriptors' - '?filter=metric.type%3Dstarts_with(%22custom.googleapis.com%2Fnetmon%2F%22)' - '&pageSize=500') +URL = ('https://content-monitoring.googleapis.com/v3/projects' + '/{}/metricDescriptors' + '?filter=metric.type%3Dstarts_with(%22custom.googleapis.com%2F{}%22)' + '&pageSize=500') def _handle_discovery(resources, response, data): @@ -51,10 +50,12 @@ def init(resources): @register_discovery(Level.CORE, 0) def start_discovery(resources, response=None, data=None): LOGGER.info(f'discovery (has response: {response is not None})') + project_id = resources['config:monitoring_project'] + type_root = resources['config:monitoring_root'] + url = URL.format(urllib.parse.quote_plus(project_id), + urllib.parse.quote_plus(type_root)) if response is None: - monitoring_project = resources['config:monitoring_project'] - yield HTTPRequest(URL.format(urllib.parse.quote_plus(monitoring_project)), - {}, None) + yield HTTPRequest(url, {}, None) else: for result in _handle_discovery(resources, response, data): yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py new file mode 100644 index 0000000000..4c549e17bd --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -0,0 +1,53 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging + +from . import HTTPRequest +from .utils import batched + +DESCRIPTOR_TYPE_BASE = 'custom.googleapis.com/{}' +DESCRIPTOR_URL = ('https://content-monitoring.googleapis.com/v3' + '/projects/{}/metricDescriptors?alt=json') +HEADERS = {'content-type': 'application/json'} +LOGGER = logging.getLogger('net-dash.plugins.monitoring') + + +def create_descriptors(project_id, root, existing, computed): + type_base = DESCRIPTOR_TYPE_BASE.format(root) + url = DESCRIPTOR_URL.format(project_id) + for descriptor in computed: + d_type = f'{type_base}{descriptor.type}' + if d_type in existing: + continue + LOGGER.info(f'creating descriptor {d_type}') + if descriptor.is_ratio: + unit = '10^2.%' + value_type = 'DOUBLE' + else: + unit = '1' + value_type = 'INT64' + data = json.dumps({ + 'type': d_type, + 'displayName': descriptor.name, + 'metricKind': 'GAUGE', + 'valueType': value_type, + 'unit': unit, + 'labels': [{ + 'key': l, + 'valueType': 'STRING' + } for l in descriptor.labels] + }) + yield HTTPRequest(url, HEADERS, data) From eaaa55abc041919b7a9ef0e9f61e20410e8fe9f1 Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 25 Nov 2022 08:41:32 +0100 Subject: [PATCH 52/82] create descriptors log --- blueprints/cloud-operations/network-dashboard/cf/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 20ea4b681c..11e0e64677 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -99,8 +99,12 @@ def do_timeseries_descriptors(project_id, existing, computed): LOGGER.info('timeseries descriptors start') urls = plugins.monitoring.create_descriptors(project_id, MONITORING_ROOT, existing, computed) + num = 0 for url in urls: fetch(url) + num += 1 + LOGGER.info('timeseries descriptors end (computed: {} created: {})'.format( + len(computed), num)) def do_timeseries(project_id, timeseries): From 9b5102f56955210d76f76c6c0def9f394c80407a Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 25 Nov 2022 08:44:37 +0100 Subject: [PATCH 53/82] rename descriptor requests function --- blueprints/cloud-operations/network-dashboard/cf/main.py | 4 ++-- .../network-dashboard/cf/plugins/monitoring.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 11e0e64677..7dd8d98323 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -97,8 +97,8 @@ def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): def do_timeseries_descriptors(project_id, existing, computed): 'Post timeseries descriptors.' LOGGER.info('timeseries descriptors start') - urls = plugins.monitoring.create_descriptors(project_id, MONITORING_ROOT, - existing, computed) + urls = plugins.monitoring.descriptor_requests(project_id, MONITORING_ROOT, + existing, computed) num = 0 for url in urls: fetch(url) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index 4c549e17bd..492a8c9a9a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -25,7 +25,7 @@ LOGGER = logging.getLogger('net-dash.plugins.monitoring') -def create_descriptors(project_id, root, existing, computed): +def descriptor_requests(project_id, root, existing, computed): type_base = DESCRIPTOR_TYPE_BASE.format(root) url = DESCRIPTOR_URL.format(project_id) for descriptor in computed: From 0511700960b0fd27a27964583dd4206dcf9af486 Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 25 Nov 2022 10:07:36 +0100 Subject: [PATCH 54/82] non-working metrics implementation (duplicate timeseries batched) --- .../network-dashboard/cf/main.py | 19 ++++++++--- .../cf/plugins/monitoring.py | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 7dd8d98323..357ea1546e 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -97,11 +97,11 @@ def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): def do_timeseries_descriptors(project_id, existing, computed): 'Post timeseries descriptors.' LOGGER.info('timeseries descriptors start') - urls = plugins.monitoring.descriptor_requests(project_id, MONITORING_ROOT, - existing, computed) + requests = plugins.monitoring.descriptor_requests(project_id, MONITORING_ROOT, + existing, computed) num = 0 - for url in urls: - fetch(url) + for request in requests: + fetch(request) num += 1 LOGGER.info('timeseries descriptors end (computed: {} created: {})'.format( len(computed), num)) @@ -110,12 +110,20 @@ def do_timeseries_descriptors(project_id, existing, computed): def do_timeseries(project_id, timeseries): 'Post timeseries.' LOGGER.info('timeseries start') + requests = plugins.monitoring.timeseries_requests(project_id, MONITORING_ROOT, + timeseries) + num = 0 + for request in requests: + # fetch(request) + num += 1 + LOGGER.info('timeseries end (computed: {} created: {})'.format( + len(timeseries), num)) def fetch(request): 'Minimal HTTP client interface for API calls.' # try - LOGGER.info(f'fetch {request.url}') + LOGGER.info(f'fetch {"POST" if request.data else "GET"} {request.url}') try: if not request.data: response = HTTP.get(request.url, headers=request.headers) @@ -127,6 +135,7 @@ def fetch(request): if response.status_code != 200: LOGGER.critical( f'response code {response.status_code} for URL {request.url}') + LOGGER.critical(response.content) return return response diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index 492a8c9a9a..9846cd0c5a 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import json import logging @@ -23,6 +24,8 @@ '/projects/{}/metricDescriptors?alt=json') HEADERS = {'content-type': 'application/json'} LOGGER = logging.getLogger('net-dash.plugins.monitoring') +TIMESERIES_URL = ('https://content-monitoring.googleapis.com/v3' + '/projects/{}/timeSeries?alt=json') def descriptor_requests(project_id, root, existing, computed): @@ -51,3 +54,32 @@ def descriptor_requests(project_id, root, existing, computed): } for l in descriptor.labels] }) yield HTTPRequest(url, HEADERS, data) + + +def timeseries_requests(project_id, root, timeseries): + end_time = ''.join((datetime.datetime.utcnow().isoformat('T'), 'Z')) + type_base = DESCRIPTOR_TYPE_BASE.format(root) + url = TIMESERIES_URL.format(project_id) + # TODO: bucketize per metric type + for batch in batched(timeseries, 190): + data = {'timeSeries': []} + for ts in batch: + pv = 'int64Value' if isinstance(ts.value, int) else 'doubleValue' + data['timeSeries'].append({ + 'metric': { + 'type': f'{type_base}{ts.metric}', + 'labels': ts.labels + }, + 'resource': { + 'type': 'global' + }, + 'points': [{ + 'interval': { + 'endTime': end_time + }, + 'value': { + pv: ts.value + } + }] + }) + yield HTTPRequest(url, HEADERS, json.dumps(data)) From 19cb6043774b669fe546d5a12d1d94638e7f60a1 Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 25 Nov 2022 17:50:55 +0100 Subject: [PATCH 55/82] timeseries --- .../network-dashboard/cf/plugins/monitoring.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index 9846cd0c5a..d47400919e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import datetime +import itertools import json import logging @@ -60,10 +62,16 @@ def timeseries_requests(project_id, root, timeseries): end_time = ''.join((datetime.datetime.utcnow().isoformat('T'), 'Z')) type_base = DESCRIPTOR_TYPE_BASE.format(root) url = TIMESERIES_URL.format(project_id) - # TODO: bucketize per metric type - for batch in batched(timeseries, 190): + ts_buckets = {} + for ts in timeseries: + bucket = ts_buckets.setdefault(ts.metric, collections.deque()) + bucket.append(ts) + ts_buckets = list(ts_buckets.values()) + while ts_buckets: data = {'timeSeries': []} - for ts in batch: + i = 0 + for bucket in ts_buckets: + ts = bucket.popleft() pv = 'int64Value' if isinstance(ts.value, int) else 'doubleValue' data['timeSeries'].append({ 'metric': { @@ -82,4 +90,7 @@ def timeseries_requests(project_id, root, timeseries): } }] }) + i += 1 + ts_buckets = [t for t in ts_buckets if t] + LOGGER.info(f'sending {i} timeseries {len(ts_buckets)} buckets left') yield HTTPRequest(url, HEADERS, json.dumps(data)) From 6b85b87d4d5221425cda9cc6cb0b294c2ade1098 Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 25 Nov 2022 20:15:01 +0100 Subject: [PATCH 56/82] fixes --- .../network-dashboard/cf/main.py | 10 +- .../network-dashboard/cf/out.json | 127 +++++------------- .../cf/plugins/discover-cai-compute.py | 2 +- .../cf/plugins/monitoring.py | 61 ++++----- .../cf/plugins/series-firewall-rules.py | 4 +- .../cf/plugins/series-subnets.py | 3 +- .../cf/tools/remove-descriptors.py | 70 ++++++++++ 7 files changed, 147 insertions(+), 130 deletions(-) create mode 100755 blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 357ea1546e..d3858c8623 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -107,14 +107,14 @@ def do_timeseries_descriptors(project_id, existing, computed): len(computed), num)) -def do_timeseries(project_id, timeseries): +def do_timeseries(project_id, timeseries, descriptors): 'Post timeseries.' LOGGER.info('timeseries start') requests = plugins.monitoring.timeseries_requests(project_id, MONITORING_ROOT, - timeseries) + timeseries, descriptors) num = 0 for request in requests: - # fetch(request) + fetch(request) num += 1 LOGGER.info('timeseries end (computed: {} created: {})'.format( len(timeseries), num)) @@ -136,6 +136,8 @@ def fetch(request): LOGGER.critical( f'response code {response.status_code} for URL {request.url}') LOGGER.critical(response.content) + print(request.data) + raise SystemExit(1) return return response @@ -180,7 +182,7 @@ def main(organization=None, op_project=None, project=None, folder=None, do_timeseries_calc(resources, descriptors, timeseries, debug_plugin) do_timeseries_descriptors(op_project, resources['metric-descriptors'], descriptors) - do_timeseries(op_project, timeseries) + do_timeseries(op_project, timeseries, descriptors) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 36f327065c..8a362121f8 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -748,7 +748,7 @@ "cidr_range": "10.24.33.0/24", "network": "projects/tf-playground-simple/global/networks/default", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-simple/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default": { "id": "3192067775436040655", @@ -759,7 +759,7 @@ "cidr_range": "10.0.0.0/24", "network": "projects/tf-playground-simple/global/networks/test", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-simple/regions/europe-west8" + "region": "europe-west8" }, "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1": { "id": "4645967677499694788", @@ -770,7 +770,7 @@ "cidr_range": "10.128.64.0/24", "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west1" + "region": "europe-west1" }, "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1": { "id": "5412553412046524467", @@ -781,7 +781,7 @@ "cidr_range": "10.128.92.0/24", "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", "purpose": "REGIONAL_MANAGED_PROXY", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west1" + "region": "europe-west1" }, "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4": { "id": "6980100425062399320", @@ -792,7 +792,7 @@ "cidr_range": "10.128.65.0/24", "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west4" + "region": "europe-west4" }, "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4": { "id": "6293569359006437427", @@ -803,7 +803,7 @@ "cidr_range": "10.128.93.0/24", "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", "purpose": "REGIONAL_MANAGED_PROXY", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-spoke-0/regions/europe-west4" + "region": "europe-west4" }, "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1": { "id": "6183073250819497646", @@ -814,7 +814,7 @@ "cidr_range": "10.128.0.0/24", "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-prod-net-landing-0/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce": { "id": "1753894793794347243", @@ -825,7 +825,7 @@ "cidr_range": "10.0.32.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke": { "id": "4511601755917967596", @@ -836,7 +836,7 @@ "cidr_range": "10.0.8.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip": { "id": "5701824999950967019", @@ -847,7 +847,7 @@ "cidr_range": "10.0.16.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net": { "id": "2784995877536981227", @@ -858,7 +858,7 @@ "cidr_range": "10.0.0.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce": { "id": "4654054937679159533", @@ -869,7 +869,7 @@ "cidr_range": "10.8.32.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west3" + "region": "europe-west3" }, "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net": { "id": "1999120736153034998", @@ -880,7 +880,7 @@ "cidr_range": "10.8.0.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west3" + "region": "europe-west3" }, "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce": { "id": "4593147538303148267", @@ -891,7 +891,7 @@ "cidr_range": "10.16.32.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west4" + "region": "europe-west4" }, "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce": { "id": "7339386710113181931", @@ -902,7 +902,7 @@ "cidr_range": "10.24.32.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb": { "id": "137392004069122283", @@ -913,7 +913,7 @@ "cidr_range": "10.255.2.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "REGIONAL_MANAGED_PROXY", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net": { "id": "3309179009467998443", @@ -924,7 +924,7 @@ "cidr_range": "10.24.0.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke": { "id": "748427805746061548", @@ -935,7 +935,7 @@ "cidr_range": "10.0.9.0/24", "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net-dr/regions/us-central1" + "region": "us-central1" }, "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1": { "id": "6460040680782777531", @@ -946,7 +946,7 @@ "cidr_range": "10.128.48.0/24", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + "region": "europe-west1" }, "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1": { "id": "2831390316161982168", @@ -957,7 +957,7 @@ "cidr_range": "10.128.32.0/24", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + "region": "europe-west1" }, "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1": { "id": "249944920601617510", @@ -968,7 +968,7 @@ "cidr_range": "10.64.0.0/24", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + "region": "europe-west1" }, "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1": { "id": "2280726911218651187", @@ -979,7 +979,7 @@ "cidr_range": "10.128.60.0/24", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", "purpose": "REGIONAL_MANAGED_PROXY", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west1" + "region": "europe-west1" }, "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4": { "id": "6599648524023664689", @@ -990,7 +990,7 @@ "cidr_range": "10.128.61.0/24", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", "purpose": "REGIONAL_MANAGED_PROXY", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west4" + "region": "europe-west4" }, "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8": { "id": "4456740759944235703", @@ -1001,7 +1001,7 @@ "cidr_range": "10.128.33.0/24", "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/ludo-dev-net-spoke-0/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce": { "id": "1129946310421719129", @@ -1012,7 +1012,7 @@ "cidr_range": "10.0.32.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke": { "id": "2478860073774111734", @@ -1023,7 +1023,7 @@ "cidr_range": "10.0.8.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip": { "id": "1416395609824617176", @@ -1034,7 +1034,7 @@ "cidr_range": "10.0.16.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net": { "id": "8424841432903474166", @@ -1045,7 +1045,7 @@ "cidr_range": "10.0.0.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west1" + "region": "europe-west1" }, "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce": { "id": "8930552426943163299", @@ -1056,7 +1056,7 @@ "cidr_range": "10.8.32.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west3" + "region": "europe-west3" }, "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net": { "id": "5182660640425322036", @@ -1067,7 +1067,7 @@ "cidr_range": "10.8.0.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west3" + "region": "europe-west3" }, "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce": { "id": "2464381379294734425", @@ -1078,7 +1078,7 @@ "cidr_range": "10.16.32.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west4" + "region": "europe-west4" }, "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce": { "id": "6421800903324885909", @@ -1089,7 +1089,7 @@ "cidr_range": "10.24.32.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb": { "id": "9159849323393344035", @@ -1100,7 +1100,7 @@ "cidr_range": "10.255.2.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "REGIONAL_MANAGED_PROXY", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net": { "id": "4872843329097064947", @@ -1111,7 +1111,7 @@ "cidr_range": "10.24.0.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc": { "id": "1816076316698934074", @@ -1122,7 +1122,7 @@ "cidr_range": "172.16.255.0/25", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE_SERVICE_CONNECT", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/europe-west8" + "region": "europe-west8" }, "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke": { "id": "440151099218547687", @@ -1133,7 +1133,7 @@ "cidr_range": "10.0.9.0/24", "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", "purpose": "PRIVATE", - "region": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/regions/us-central1" + "region": "us-central1" } }, "routers": { @@ -2720,58 +2720,5 @@ "projects/tf-playground-svpc-net/global/networks/shared-vpc" ] }, - "metric-descriptors": { - "custom.googleapis.com/netmon/firewall_policies/tuples_available": {}, - "custom.googleapis.com/netmon/firewall_policies/tuples_used": {}, - "custom.googleapis.com/netmon/firewall_policies/tuples_used_ratio": {}, - "custom.googleapis.com/netmon/network/firewall_rules_available": {}, - "custom.googleapis.com/netmon/network/firewall_rules_used": {}, - "custom.googleapis.com/netmon/network/firewall_rules_used_ratio": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l4_available": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l4_used": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l7_available": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l7_used": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l7_used_ratio": {}, - "custom.googleapis.com/netmon/network/instances_available": {}, - "custom.googleapis.com/netmon/network/instances_used": {}, - "custom.googleapis.com/netmon/network/instances_used_ratio": {}, - "custom.googleapis.com/netmon/network/peerings_active_available": {}, - "custom.googleapis.com/netmon/network/peerings_active_used": {}, - "custom.googleapis.com/netmon/network/peerings_active_used_ratio": {}, - "custom.googleapis.com/netmon/network/peerings_total_available": {}, - "custom.googleapis.com/netmon/network/peerings_total_used": {}, - "custom.googleapis.com/netmon/network/peerings_total_used_ratio": {}, - "custom.googleapis.com/netmon/network/routes_dynamic_available": {}, - "custom.googleapis.com/netmon/network/routes_dynamic_used": {}, - "custom.googleapis.com/netmon/network/routes_dynamic_used_ratio": {}, - "custom.googleapis.com/netmon/network/routes_static_used": {}, - "custom.googleapis.com/netmon/network/subnets_available": {}, - "custom.googleapis.com/netmon/network/subnets_used": {}, - "custom.googleapis.com/netmon/network/subnets_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_available": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_available": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/instances_available": {}, - "custom.googleapis.com/netmon/peering_group/instances_used": {}, - "custom.googleapis.com/netmon/peering_group/instances_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/routes_dynamic_available": {}, - "custom.googleapis.com/netmon/peering_group/routes_dynamic_used": {}, - "custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/routes_static_available": {}, - "custom.googleapis.com/netmon/peering_group/routes_static_used": {}, - "custom.googleapis.com/netmon/peering_group/routes_static_used_ratio": {}, - "custom.googleapis.com/netmon/project/firewall_rules_available": {}, - "custom.googleapis.com/netmon/project/firewall_rules_used": {}, - "custom.googleapis.com/netmon/project/firewall_rules_used_ratio": {}, - "custom.googleapis.com/netmon/project/routes_dynamic_available": {}, - "custom.googleapis.com/netmon/project/routes_dynamic_used": {}, - "custom.googleapis.com/netmon/project/routes_dynamic_used_ratio": {}, - "custom.googleapis.com/netmon/subnetwork/addresses_available": {}, - "custom.googleapis.com/netmon/subnetwork/addresses_used": {}, - "custom.googleapis.com/netmon/subnetwork/addresses_used_ratio": {} - } + "metric-descriptors": {} } \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 2192b44ce2..52a95487cf 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -166,7 +166,7 @@ def _handle_subnetworks(resource, data): 'cidr_range': data['ipCidrRange'], 'network': _self_link(data['network']), 'purpose': data.get('purpose'), - 'region': data['region'] + 'region': data['region'].split('/')[-1] } diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index d47400919e..d6e4f4b9d6 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -58,39 +58,36 @@ def descriptor_requests(project_id, root, existing, computed): yield HTTPRequest(url, HEADERS, data) -def timeseries_requests(project_id, root, timeseries): +def timeseries_requests(project_id, root, timeseries, descriptors): + descriptor_valuetypes = {d.type: d.is_ratio for d in descriptors} end_time = ''.join((datetime.datetime.utcnow().isoformat('T'), 'Z')) type_base = DESCRIPTOR_TYPE_BASE.format(root) url = TIMESERIES_URL.format(project_id) - ts_buckets = {} - for ts in timeseries: - bucket = ts_buckets.setdefault(ts.metric, collections.deque()) - bucket.append(ts) - ts_buckets = list(ts_buckets.values()) - while ts_buckets: - data = {'timeSeries': []} - i = 0 - for bucket in ts_buckets: - ts = bucket.popleft() - pv = 'int64Value' if isinstance(ts.value, int) else 'doubleValue' - data['timeSeries'].append({ - 'metric': { - 'type': f'{type_base}{ts.metric}', - 'labels': ts.labels - }, - 'resource': { - 'type': 'global' - }, - 'points': [{ - 'interval': { - 'endTime': end_time - }, - 'value': { - pv: ts.value - } - }] - }) - i += 1 - ts_buckets = [t for t in ts_buckets if t] - LOGGER.info(f'sending {i} timeseries {len(ts_buckets)} buckets left') + timeseries = collections.deque(timeseries) + while timeseries: + ts = timeseries.popleft() + if descriptor_valuetypes[ts.metric]: + pv = 'doubleValue' + else: + pv = 'int64Value' + data = { + 'timeSeries': [{ + 'metric': { + 'type': f'{type_base}{ts.metric}', + 'labels': ts.labels + }, + 'resource': { + 'type': 'global' + }, + 'points': [{ + 'interval': { + 'endTime': end_time + }, + 'value': { + pv: ts.value + } + }] + }] + } + LOGGER.info(f'sending 1/{len(timeseries)} timeseries') yield HTTPRequest(url, HEADERS, json.dumps(data)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index db250f8412..67634ec306 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -33,7 +33,7 @@ def timeseries(resources): for dtype, name in DESCRIPTOR_ATTRS.items(): labels = ('project',) if prefix == 'project' else ('project', 'name') yield MetricDescriptor(f'{prefix}/{dtype}', name.format(prefix), labels, - name.endswith('ratio')) + dtype.endswith('ratio')) grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['network']) for network_id, rules in grouped: @@ -42,7 +42,7 @@ def timeseries(resources): 'name': resources['networks'][network_id]['name'], 'project': resources['networks'][network_id]['project_id'] } - yield TimeSeries('network/firewall_rules/used', count, labels) + yield TimeSeries('network/firewall_rules_used', count, labels) grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['project_id']) for project_id, rules in grouped: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index c1313aba6a..b2cc6b4811 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -68,7 +68,7 @@ def timeseries(resources): LOGGER.info('timeseries') for dtype, name in DESCRIPTOR_ATTRS.items(): yield MetricDescriptor(f'subnetwork/{dtype}', name, - ('project', 'network', 'subnetwork'), + ('project', 'network', 'subnetwork', 'region'), dtype.endswith('ratio')) subnet_nets = { k: ipaddress.ip_network(v['cidr_range']) @@ -87,6 +87,7 @@ def timeseries(resources): labels = { 'network': resources['networks'][subnet['network']]['name'], 'project': subnet['project_id'], + 'region': subnet['region'], 'subnetwork': subnet['name'] } yield TimeSeries('subnetwork/addresses_available', max_ips, labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py b/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py new file mode 100755 index 0000000000..5f639c302e --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging + +import click +import google.auth + +from google.auth.transport.requests import AuthorizedSession + +HEADERS = {'content-type': 'application/json'} +HTTP = AuthorizedSession(google.auth.default()[0]) +URL_DELETE = 'https://monitoring.googleapis.com/v3/{}' +URL_LIST = ( + 'https://monitoring.googleapis.com/v3/projects/{}' + '/metricDescriptors?filter=metric.type=starts_with("custom.googleapis.com/netmon/")' + '&alt=json') + + +def fetch(url, delete=False): + 'Minimal HTTP client interface for API calls.' + # try + try: + if not delete: + response = HTTP.get(url, headers=HEADERS) + else: + response = HTTP.delete(url) + except google.auth.exceptions.RefreshError as e: + raise SystemExit(e.args[0]) + if response.status_code != 200: + logging.critical(f'response code {response.status_code} for URL {url}') + logging.critical(response.content) + return + return response.json() + + +@click.command() +@click.option('--op-project', '-op', required=True, type=str, + help='GCP monitoring project where metrics will be stored.') +def main(op_project): + # if not click.confirm('Do you want to continue?'): + # raise SystemExit(0) + logging.info('fetching descriptors') + result = fetch(URL_LIST.format(op_project)) + descriptors = result.get('metricDescriptors') + if not descriptors: + raise SystemExit(0) + logging.info(f'{len(descriptors)} descriptors') + for d in descriptors: + name = d['name'] + logging.info(f'delete {name}') + result = fetch(URL_DELETE.format(name), True) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() \ No newline at end of file From 0dadd4a9dda1bafd0163260f0c0ee8554034559b Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 26 Nov 2022 07:08:53 +0100 Subject: [PATCH 57/82] write timseries --- .../network-dashboard/cf/main.py | 2 +- .../network-dashboard/cf/out.json | 55 +++++++++++++++- .../cf/plugins/monitoring.py | 62 +++++++++++-------- 3 files changed, 90 insertions(+), 29 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index d3858c8623..b79abd8755 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -116,7 +116,7 @@ def do_timeseries(project_id, timeseries, descriptors): for request in requests: fetch(request) num += 1 - LOGGER.info('timeseries end (computed: {} created: {})'.format( + LOGGER.info('timeseries end (number: {} requests: {})'.format( len(timeseries), num)) diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 8a362121f8..7bfa2aecfd 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -2720,5 +2720,58 @@ "projects/tf-playground-svpc-net/global/networks/shared-vpc" ] }, - "metric-descriptors": {} + "metric-descriptors": { + "custom.googleapis.com/netmon/firewall_policies/tuples_available": {}, + "custom.googleapis.com/netmon/firewall_policies/tuples_used": {}, + "custom.googleapis.com/netmon/firewall_policies/tuples_used_ratio": {}, + "custom.googleapis.com/netmon/network/firewall_rules_available": {}, + "custom.googleapis.com/netmon/network/firewall_rules_used": {}, + "custom.googleapis.com/netmon/network/firewall_rules_used_ratio": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l4_available": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l4_used": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l7_available": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l7_used": {}, + "custom.googleapis.com/netmon/network/forwarding_rules_l7_used_ratio": {}, + "custom.googleapis.com/netmon/network/instances_available": {}, + "custom.googleapis.com/netmon/network/instances_used": {}, + "custom.googleapis.com/netmon/network/instances_used_ratio": {}, + "custom.googleapis.com/netmon/network/peerings_active_available": {}, + "custom.googleapis.com/netmon/network/peerings_active_used": {}, + "custom.googleapis.com/netmon/network/peerings_active_used_ratio": {}, + "custom.googleapis.com/netmon/network/peerings_total_available": {}, + "custom.googleapis.com/netmon/network/peerings_total_used": {}, + "custom.googleapis.com/netmon/network/peerings_total_used_ratio": {}, + "custom.googleapis.com/netmon/network/routes_dynamic_available": {}, + "custom.googleapis.com/netmon/network/routes_dynamic_used": {}, + "custom.googleapis.com/netmon/network/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/network/routes_static_used": {}, + "custom.googleapis.com/netmon/network/subnets_available": {}, + "custom.googleapis.com/netmon/network/subnets_used": {}, + "custom.googleapis.com/netmon/network/subnets_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_available": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_available": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used": {}, + "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/instances_available": {}, + "custom.googleapis.com/netmon/peering_group/instances_used": {}, + "custom.googleapis.com/netmon/peering_group/instances_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/routes_dynamic_available": {}, + "custom.googleapis.com/netmon/peering_group/routes_dynamic_used": {}, + "custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/peering_group/routes_static_available": {}, + "custom.googleapis.com/netmon/peering_group/routes_static_used": {}, + "custom.googleapis.com/netmon/peering_group/routes_static_used_ratio": {}, + "custom.googleapis.com/netmon/project/firewall_rules_available": {}, + "custom.googleapis.com/netmon/project/firewall_rules_used": {}, + "custom.googleapis.com/netmon/project/firewall_rules_used_ratio": {}, + "custom.googleapis.com/netmon/project/routes_dynamic_available": {}, + "custom.googleapis.com/netmon/project/routes_dynamic_used": {}, + "custom.googleapis.com/netmon/project/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/subnetwork/addresses_available": {}, + "custom.googleapis.com/netmon/subnetwork/addresses_used": {}, + "custom.googleapis.com/netmon/subnetwork/addresses_used_ratio": {} + } } \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index d6e4f4b9d6..24e4986817 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -50,6 +50,7 @@ def descriptor_requests(project_id, root, existing, computed): 'metricKind': 'GAUGE', 'valueType': value_type, 'unit': unit, + 'monitoredResourceTypes': ['global'], 'labels': [{ 'key': l, 'valueType': 'STRING' @@ -63,31 +64,38 @@ def timeseries_requests(project_id, root, timeseries, descriptors): end_time = ''.join((datetime.datetime.utcnow().isoformat('T'), 'Z')) type_base = DESCRIPTOR_TYPE_BASE.format(root) url = TIMESERIES_URL.format(project_id) - timeseries = collections.deque(timeseries) - while timeseries: - ts = timeseries.popleft() - if descriptor_valuetypes[ts.metric]: - pv = 'doubleValue' - else: - pv = 'int64Value' - data = { - 'timeSeries': [{ - 'metric': { - 'type': f'{type_base}{ts.metric}', - 'labels': ts.labels - }, - 'resource': { - 'type': 'global' - }, - 'points': [{ - 'interval': { - 'endTime': end_time - }, - 'value': { - pv: ts.value - } - }] - }] - } - LOGGER.info(f'sending 1/{len(timeseries)} timeseries') + ts_buckets = {} + for ts in timeseries: + bucket = ts_buckets.setdefault(ts.metric, collections.deque()) + bucket.append(ts) + ts_buckets = list(ts_buckets.values()) + while ts_buckets: + data = {'timeSeries': []} + for bucket in ts_buckets: + ts = bucket.popleft() + if descriptor_valuetypes[ts.metric]: + pv = 'doubleValue' + else: + pv = 'int64Value' + data['timeSeries'].append({ + 'metric': { + 'type': f'{type_base}{ts.metric}', + 'labels': ts.labels + }, + 'resource': { + 'type': 'global' + }, + 'points': [{ + 'interval': { + 'endTime': end_time + }, + 'value': { + pv: ts.value + } + }] + }) + req_num = len(data['timeSeries']) + tot_num = sum(len(b) for b in ts_buckets) + LOGGER.info(f'sending {req_num} remaining:\ {tot_num}') yield HTTPRequest(url, HEADERS, json.dumps(data)) + ts_buckets = [b for b in ts_buckets if b] \ No newline at end of file From 4628bd46bf990fa5be7d9b3feb6086d080080e1e Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 26 Nov 2022 07:35:38 +0100 Subject: [PATCH 58/82] fix timeseries plugins --- .../cloud-operations/network-dashboard/cf/main.py | 4 ++++ .../network-dashboard/cf/plugins/monitoring.py | 4 ++-- .../network-dashboard/cf/plugins/series-networks.py | 6 ++++-- .../cf/plugins/series-peering-groups.py | 6 ++++-- .../network-dashboard/cf/plugins/series-routes.py | 12 ++++++++++-- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index b79abd8755..d48c14f6c4 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -83,13 +83,17 @@ def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): if debug_plugin and plugin.name != debug_plugin: LOGGER.info(f'skipping {plugin.name}') continue + num_desc, num_ts = 0, 0 for result in plugin.func(resources): if not result: continue if isinstance(result, plugins.MetricDescriptor): descriptors.append(result) + num_desc += 1 elif isinstance(result, plugins.TimeSeries): timeseries.append(result) + num_ts += 1 + LOGGER.info(f'{plugin.name}: {num_desc} descriptors {num_ts} timeseries') LOGGER.info('timeseries calc end (descriptors: {} timeseries: {})'.format( len(descriptors), len(timeseries))) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index 24e4986817..922a244335 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -14,7 +14,6 @@ import collections import datetime -import itertools import json import logging @@ -68,6 +67,7 @@ def timeseries_requests(project_id, root, timeseries, descriptors): for ts in timeseries: bucket = ts_buckets.setdefault(ts.metric, collections.deque()) bucket.append(ts) + LOGGER.info(f'metric types {list(ts_buckets.keys())}') ts_buckets = list(ts_buckets.values()) while ts_buckets: data = {'timeSeries': []} @@ -96,6 +96,6 @@ def timeseries_requests(project_id, root, timeseries, descriptors): }) req_num = len(data['timeSeries']) tot_num = sum(len(b) for b in ts_buckets) - LOGGER.info(f'sending {req_num} remaining:\ {tot_num}') + LOGGER.info(f'sending {req_num} remaining: {tot_num}') yield HTTPRequest(url, HEADERS, json.dumps(data)) ts_buckets = [b for b in ts_buckets if b] \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index 986d6e9ac2..ef66723c98 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -124,5 +124,7 @@ def timeseries(resources): for dtype, name in DESCRIPTOR_ATTRS.items(): yield MetricDescriptor(f'network/{dtype}', name, ('project', 'network'), dtype.endswith('ratio')) - return itertools.chain(_forwarding_rules(resources), _instances(resources), - _peerings(resources), _subnet_ranges(resources)) + results = itertools.chain(_forwarding_rules(resources), _instances(resources), + _peerings(resources), _subnet_ranges(resources)) + for result in results: + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py index 4f2f2c2311..4778b00f45 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py @@ -163,5 +163,7 @@ def timeseries(resources): for dtype, name in DESCRIPTOR_ATTRS.items(): yield MetricDescriptor(f'peering_group/{dtype}', name, ('project', 'network'), dtype.endswith('ratio')) - return itertools.chain(*(_network_timeseries(resources, n) - for n in resources['networks'].values())) + results = itertools.chain(*(_network_timeseries(resources, n) + for n in resources['networks'].values())) + for result in results: + yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py index 8a97222323..83764d024f 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py @@ -31,7 +31,13 @@ 'project/routes_dynamic_available': 'Dynamic routes used per project', 'project/routes_dynamic_used_ratio': - 'Dynamic routes used ratio per project' + 'Dynamic routes used ratio per project', + 'project/routes_static_used': + 'Static routes limit per project', + 'project/routes_static_available': + 'Static routes used per project', + 'project/routes_static_used_ratio': + 'Static routes used ratio per project' } LIMITS = {'ROUTES': 250, 'ROUTES_DYNAMIC': 100} LOGGER = logging.getLogger('net-dash.timeseries.routes') @@ -79,4 +85,6 @@ def timeseries(resources): labels = ('project') if dtype.startswith('project') else ('project', 'network') yield MetricDescriptor(dtype, name, labels, dtype.endswith('ratio')) - return itertools.chain(_static(resources), _dynamic(resources)) + results = itertools.chain(_static(resources), _dynamic(resources)) + for result in results: + yield result From 651a893839b2b892f175181675a8516b21e54a6a Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 26 Nov 2022 09:08:43 +0100 Subject: [PATCH 59/82] start documenting code --- .../network-dashboard/cf/main.py | 69 +++++++++++++++++-- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index d48c14f6c4..561b7b5e5f 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Network dashboard: create network-related metric timeseries for GCP resources.' import collections import json @@ -33,28 +34,43 @@ def do_discovery(resources): - 'Discover resources needed in measurements.' + '''Calls discovery plugin functions and collect discovered resources. + + The communication with discovery plugins uses double dispatch, where plugins + accept either no args and return 1-n HTTP request instances, or a single HTTP + response and return 1-n resource instances. A queue is set up for each plugin + results since each call can return multiple requests or resources. + + Args: + resources: pre-initialized map where discovered resources will be stored. + ''' LOGGER.info(f'discovery start') for plugin in plugins.get_discovery_plugins(): + # set up the queue with the initial list of HTTP requests from this plugin q = collections.deque(plugin.func(resources)) while q: result = q.popleft() if isinstance(result, plugins.HTTPRequest): + # fetch a single HTTP request response = fetch(result) if not response: continue if result.json: try: + # decode the JSON HTTP response and pass it to the plugin results = plugin.func(resources, response, response.json()) except json.decoder.JSONDecodeError as e: LOGGER.critical( f'error decoding JSON for {result.url}: {e.args[0]}') continue else: + # pass the raw HTTP response to the plugin results = plugin.func(resources, response) q += collections.deque(results) elif isinstance(result, plugins.Resource): + # store a resource the plugin derived from a previous HTTP response if result.key: + # this specific resource is indexed by an additional key resources[result.type][result.id][result.key] = result.data else: resources[result.type][result.id] = result.data @@ -64,7 +80,14 @@ def do_discovery(resources): def do_init(resources, organization, op_project, folders=None, projects=None, custom_quota=None): - 'Prepare the resources datastructure fields.' + '''Calls init plugins to configure keys in the shared resource map. + + Args: + organization: organization id from configuration. + op_project: monitoring project id id from configuration. + folders: list of folder ids for resource discovery from configuration. + projects: list of project ids for resource discovery from configuration. + ''' LOGGER.info(f'init start') resources['config:organization'] = str(organization) resources['config:monitoring_project'] = op_project @@ -77,7 +100,17 @@ def do_init(resources, organization, op_project, folders=None, projects=None, def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): - 'Calculate descriptors and timeseries.' + '''Calls timeseries plugins and collect resulting descriptors and timeseries. + + Timeseries plugin return a list of MetricDescriptors and Timeseries instances, + one per each metric. + + Args: + resources: shared map of configuration and discovered resources. + descriptors: list where collected descriptors will be stored. + timeseries: list where collected timeseries will be stored. + debug_plugin: optional name of a single plugin to call + ''' LOGGER.info(f'timeseries calc start (debug plugin: {debug_plugin})') for plugin in plugins.get_timeseries_plugins(): if debug_plugin and plugin.name != debug_plugin: @@ -87,6 +120,7 @@ def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): for result in plugin.func(resources): if not result: continue + # append result to the relevant collection (descriptors or timeseries) if isinstance(result, plugins.MetricDescriptor): descriptors.append(result) num_desc += 1 @@ -99,7 +133,13 @@ def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): def do_timeseries_descriptors(project_id, existing, computed): - 'Post timeseries descriptors.' + '''Executes API calls for each previously computed metric descriptor. + + Args: + project_id: monitoring project id where to write descriptors. + existing: map of existing descriptor types. + computed: list of plugins.MetricDescriptor instances previously computed. + ''' LOGGER.info('timeseries descriptors start') requests = plugins.monitoring.descriptor_requests(project_id, MONITORING_ROOT, existing, computed) @@ -112,7 +152,13 @@ def do_timeseries_descriptors(project_id, existing, computed): def do_timeseries(project_id, timeseries, descriptors): - 'Post timeseries.' + '''Executes API calls for each previously computed timeseries. + + Args: + project_id: monitoring project id where to write timeseries. + timeseries: list of plugins.Timeseries instances. + descriptors: list of plugins.MetricDescriptor instances matching timeseries. + ''' LOGGER.info('timeseries start') requests = plugins.monitoring.timeseries_requests(project_id, MONITORING_ROOT, timeseries, descriptors) @@ -125,7 +171,16 @@ def do_timeseries(project_id, timeseries, descriptors): def fetch(request): - 'Minimal HTTP client interface for API calls.' + '''Minimal HTTP client interface for API calls. + + Executes the HTTP request passed as argument using the google.auth + authenticated session. + + Args: + request: an instance of plugins.HTTPRequest. + Returns: + JSON-decoded or raw response depending on the 'json' request attribute. + ''' # try LOGGER.info(f'fetch {"POST" if request.data else "GET"} {request.url}') try: @@ -142,7 +197,6 @@ def fetch(request): LOGGER.critical(response.content) print(request.data) raise SystemExit(1) - return return response @@ -166,6 +220,7 @@ def fetch(request): def main(organization=None, op_project=None, project=None, folder=None, custom_quota_file=None, dump_file=None, load_file=None, debug_plugin=None): + 'CLI entry point.' logging.basicConfig(level=logging.INFO) descriptors = [] timeseries = [] From 67665b81fa2a38e3385da7353e17c57df86328bb Mon Sep 17 00:00:00 2001 From: Ludo Date: Sat, 26 Nov 2022 16:45:37 +0100 Subject: [PATCH 60/82] docstrings and comments --- .../network-dashboard/cf/plugins/__init__.py | 6 +++ ...jects.py => core-discover-cai-projects.py} | 12 ++++++ .../cf/plugins/discover-cai-compute.py | 41 ++++++++++++------- 3 files changed, 44 insertions(+), 15 deletions(-) rename blueprints/cloud-operations/network-dashboard/cf/plugins/{discover-cai-projects.py => core-discover-cai-projects.py} (77%) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index b1244392ea..09c1dbd0ac 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -11,6 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Plugin interface objects and registration functions. + +This module export the objects passed to and returned from plugin functions, +and the function used to register plugins for each stage, and get all plugins +for individual stages. +''' import collections import enum diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/core-discover-cai-projects.py similarity index 77% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py rename to blueprints/cloud-operations/network-dashboard/cf/plugins/core-discover-cai-projects.py index 862f99e92e..5705ded99c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/core-discover-cai-projects.py @@ -11,6 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Project discovery from configuration options. + +This plugin needs to run first, as it's responsible for discovering projects +and their attributes, based on configuration options. Projects are fetched +from Cloud Asset Inventory based on explicit id or being part of a folder +hierarchy. +''' import logging @@ -28,6 +35,7 @@ def _handle_discovery(resources, response, data): + 'Processes asset response and returns project resources or next URLs.' LOGGER.info('discovery handle request') for result in parse_cai_results(data, NAME, TYPE): data = { @@ -44,6 +52,7 @@ def _handle_discovery(resources, response, data): @register_init def init(resources): + 'Prepares project datastructures in the shared resource map.' LOGGER.info('init') resources.setdefault(NAME, {}) resources.setdefault('projects:number', {}) @@ -51,12 +60,15 @@ def init(resources): @register_discovery(Level.CORE, 0) def start_discovery(resources, response=None, data=None): + 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') if response is None: + # return asset discovery URLs from initial options on first call for v in resources['config:projects']: yield HTTPRequest(CAI_URL.format(f'projects/{v}'), {}, None) for v in resources['config:folders']: yield HTTPRequest(CAI_URL.format(f'folders/{v}'), {}, None) else: + # pass the API response to the plugin data handler and return results for result in _handle_discovery(resources, response, data): yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py index 52a95487cf..59283673c3 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py @@ -11,6 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Compute resources discovery from Cloud Asset Inventory. + +This plugin handles discovery for Compute resources via a broad org-level +scoped CAI search. Common resource attributes are parsed by a generic handler +function, which then delegates parsing of resource-level attributes to smaller +specialized functions, one per resource type. +''' import logging @@ -38,7 +45,7 @@ def _handle_discovery(resources, response, data): - 'Process discovery data.' + 'Processes the asset API response and returns parsed resources or next URL.' LOGGER.info('discovery handle request') for result in parse_cai_results(data, 'cai-compute', method='list'): resource = _handle_resource(resources, result['resource']) @@ -53,18 +60,22 @@ def _handle_discovery(resources, response, data): def _handle_resource(resources, data): + 'Parses and returns a single resource. Calls resource-level handler.' attrs = data['data'] + # general attributes shared by all resource types resource_name = NAMES[data['discoveryName']] resource = { 'id': attrs['id'], 'name': attrs['name'], 'self_link': _self_link(attrs['selfLink']) } + # derive parent type and id and skip if parent is not within scope parent_data = _get_parent(data['parent'], resources) if not parent_data: LOGGER.info(f'{resource["self_link"]} outside perimeter') return resource.update(parent_data) + # gets and calls the resource-level handler for type specific attributes func = globals().get(f'_handle_{resource_name}') if not callable(func): raise SystemExit(f'specialized function missing for {resource_name}') @@ -76,7 +87,7 @@ def _handle_resource(resources, data): def _handle_addresses(resource, data): - 'Handle address type resource data.' + 'Handles address type resource data.' network = data.get('network') subnet = data.get('subnetwork') return { @@ -90,7 +101,7 @@ def _handle_addresses(resource, data): def _handle_firewall_policies(resource, data): - 'Handle firewall policy type resource data.' + 'Handles firewall policy type resource data.' return { 'num_rules': len(data.get('rules', [])), 'num_tuples': data.get('ruleTupleCount', 0) @@ -98,12 +109,12 @@ def _handle_firewall_policies(resource, data): def _handle_firewall_rules(resource, data): - 'Handle firewall type resource data.' + 'Handles firewall type resource data.' return {'network': _self_link(data['network'])} def _handle_forwarding_rules(resource, data): - 'Handle forwarding_rules type resource data.' + 'Handles forwarding_rules type resource data.' network = data.get('network') region = data.get('region') subnet = data.get('subnetwork') @@ -118,7 +129,7 @@ def _handle_forwarding_rules(resource, data): def _handle_instances(resource, data): - 'Handle instance type resource data.' + 'Handles instance type resource data.' if data['status'] != 'RUNNING': return networks = [{ @@ -129,7 +140,7 @@ def _handle_instances(resource, data): def _handle_networks(resource, data): - 'Handle network type resource data.' + 'Handles network type resource data.' peerings = [{ 'active': p['state'] == 'ACTIVE', 'name': p['name'], @@ -141,7 +152,7 @@ def _handle_networks(resource, data): def _handle_routers(resource, data): - 'Handle router type resource data.' + 'Handles router type resource data.' return { 'network': _self_link(data['network']), 'region': data['region'].split('/')[-1] @@ -149,7 +160,7 @@ def _handle_routers(resource, data): def _handle_routes(resource, data): - 'Handle route type resource data.' + 'Handles route type resource data.' hop = [ a.removeprefix('nextHop').lower() for a in data if a.startswith('nextHop') ] @@ -157,7 +168,7 @@ def _handle_routes(resource, data): def _handle_subnetworks(resource, data): - 'Handle subnetwork type resource data.' + 'Handles subnetwork type resource data.' secondary_ranges = [{ 'name': s['rangeName'], 'cidr_range': s['ipCidrRange'] @@ -171,12 +182,12 @@ def _handle_subnetworks(resource, data): def _self_link(s): - 'Remove initial part from self links.' + 'Removes initial part from self links.' return s.removeprefix('https://www.googleapis.com/compute/v1/') def _get_parent(parent, resources): - 'Extract and return resource parent.' + 'Extracts and returns resource parent and type.' parent_type, parent_id = parent.split('/')[-2:] if parent_type == 'projects': project = resources['projects:number'].get(parent_id) @@ -190,7 +201,7 @@ def _get_parent(parent, resources): def _url(resources): - 'Return discovery URL' + 'Returns discovery URL' organization = resources['config:organization'] asset_types = '&'.join( f'assetTypes=compute.googleapis.com/{t}' for t in TYPES.values()) @@ -199,7 +210,7 @@ def _url(resources): @register_init def init(resources): - 'Prepare the shared datastructures for asset types managed here.' + 'Prepares the datastructures for types managed here in the resource map.' LOGGER.info('init') for name in TYPES: resources.setdefault(name, {}) @@ -207,7 +218,7 @@ def init(resources): @register_discovery(Level.PRIMARY, 10) def start_discovery(resources, response=None, data=None): - 'Start discovery by returning the asset list URL for asset types.' + 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') if response is None: yield HTTPRequest(_url(resources), {}, None) From 97c4c31742381da0120f4164df043082cd74ed01 Mon Sep 17 00:00:00 2001 From: Ludo Date: Sun, 27 Nov 2022 10:24:11 +0100 Subject: [PATCH 61/82] docstrings comments and small fixes --- .../network-dashboard/cf/main.py | 2 +- .../network-dashboard/cf/out.json | 227 +++++++++--------- .../network-dashboard/cf/plugins/__init__.py | 3 +- .../cf/plugins/discover-compute-quota.py | 18 +- .../plugins/discover-compute-routerstatus.py | 17 +- .../cf/plugins/discover-group-networks.py | 3 + .../cf/plugins/discover-metric-descriptors.py | 8 + .../cf/plugins/monitoring.py | 7 +- .../cf/plugins/series-firewall-policies.py | 16 +- .../cf/plugins/series-firewall-rules.py | 23 +- .../cf/plugins/series-networks.py | 24 +- .../cf/plugins/series-peering-groups.py | 27 ++- .../cf/plugins/series-routes.py | 9 +- .../cf/plugins/series-subnets.py | 14 +- .../network-dashboard/cf/plugins/utils.py | 55 ++--- .../cf/tools/remove-descriptors.py | 2 + 16 files changed, 265 insertions(+), 190 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/cf/main.py index 561b7b5e5f..7da320c2eb 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/main.py +++ b/blueprints/cloud-operations/network-dashboard/cf/main.py @@ -182,7 +182,7 @@ def fetch(request): JSON-decoded or raw response depending on the 'json' request attribute. ''' # try - LOGGER.info(f'fetch {"POST" if request.data else "GET"} {request.url}') + LOGGER.debug(f'fetch {"POST" if request.data else "GET"} {request.url}') try: if not request.data: response = HTTP.get(request.url, headers=request.headers) diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/cf/out.json index 7bfa2aecfd..52a533f263 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/out.json +++ b/blueprints/cloud-operations/network-dashboard/cf/out.json @@ -24,6 +24,114 @@ } }, "config:monitoring_root": "netmon/", + "projects": { + "tf-playground-simple": { + "number": "64297462517", + "project_id": "tf-playground-simple" + }, + "ludo-dev-net-spoke-0": { + "number": "759592912116", + "project_id": "ludo-dev-net-spoke-0" + }, + "ludo-prod-net-landing-0": { + "number": "233262889141", + "project_id": "ludo-prod-net-landing-0" + }, + "ludo-prod-net-spoke-0": { + "number": "195159130008", + "project_id": "ludo-prod-net-spoke-0" + }, + "ludo-dev-sec-core-0": { + "number": "971935141727", + "project_id": "ludo-dev-sec-core-0" + }, + "ludo-prod-sec-core-0": { + "number": "990827422843", + "project_id": "ludo-prod-sec-core-0" + }, + "tf-playground-gcs-test-0": { + "number": "833078410166", + "project_id": "tf-playground-gcs-test-0" + }, + "tf-playground-svpc-gce-dr": { + "number": "433746214138", + "project_id": "tf-playground-svpc-gce-dr" + }, + "tf-playground-svpc-net-dr": { + "number": "697669426824", + "project_id": "tf-playground-svpc-net-dr" + }, + "tf-playground-svpc-openshift": { + "number": "515444627958", + "project_id": "tf-playground-svpc-openshift" + }, + "tf-playground-svpc-gce": { + "number": "783093469136", + "project_id": "tf-playground-svpc-gce" + }, + "tf-playground-svpc-net": { + "number": "1079408472053", + "project_id": "tf-playground-svpc-net" + }, + "tf-playground-svpc-gke": { + "number": "1043465648801", + "project_id": "tf-playground-svpc-gke" + } + }, + "projects:number": { + "64297462517": { + "number": "64297462517", + "project_id": "tf-playground-simple" + }, + "759592912116": { + "number": "759592912116", + "project_id": "ludo-dev-net-spoke-0" + }, + "233262889141": { + "number": "233262889141", + "project_id": "ludo-prod-net-landing-0" + }, + "195159130008": { + "number": "195159130008", + "project_id": "ludo-prod-net-spoke-0" + }, + "971935141727": { + "number": "971935141727", + "project_id": "ludo-dev-sec-core-0" + }, + "990827422843": { + "number": "990827422843", + "project_id": "ludo-prod-sec-core-0" + }, + "833078410166": { + "number": "833078410166", + "project_id": "tf-playground-gcs-test-0" + }, + "433746214138": { + "number": "433746214138", + "project_id": "tf-playground-svpc-gce-dr" + }, + "697669426824": { + "number": "697669426824", + "project_id": "tf-playground-svpc-net-dr" + }, + "515444627958": { + "number": "515444627958", + "project_id": "tf-playground-svpc-openshift" + }, + "783093469136": { + "number": "783093469136", + "project_id": "tf-playground-svpc-gce" + }, + "1079408472053": { + "number": "1079408472053", + "project_id": "tf-playground-svpc-net" + }, + "1043465648801": { + "number": "1043465648801", + "project_id": "tf-playground-svpc-gke" + } + }, "addresses": { "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello": { "id": "2569728380045293941", @@ -1995,114 +2103,6 @@ "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" } }, - "projects": { - "tf-playground-simple": { - "number": "64297462517", - "project_id": "tf-playground-simple" - }, - "ludo-dev-net-spoke-0": { - "number": "759592912116", - "project_id": "ludo-dev-net-spoke-0" - }, - "ludo-prod-net-landing-0": { - "number": "233262889141", - "project_id": "ludo-prod-net-landing-0" - }, - "ludo-prod-net-spoke-0": { - "number": "195159130008", - "project_id": "ludo-prod-net-spoke-0" - }, - "ludo-dev-sec-core-0": { - "number": "971935141727", - "project_id": "ludo-dev-sec-core-0" - }, - "ludo-prod-sec-core-0": { - "number": "990827422843", - "project_id": "ludo-prod-sec-core-0" - }, - "tf-playground-gcs-test-0": { - "number": "833078410166", - "project_id": "tf-playground-gcs-test-0" - }, - "tf-playground-svpc-gce-dr": { - "number": "433746214138", - "project_id": "tf-playground-svpc-gce-dr" - }, - "tf-playground-svpc-net-dr": { - "number": "697669426824", - "project_id": "tf-playground-svpc-net-dr" - }, - "tf-playground-svpc-openshift": { - "number": "515444627958", - "project_id": "tf-playground-svpc-openshift" - }, - "tf-playground-svpc-gce": { - "number": "783093469136", - "project_id": "tf-playground-svpc-gce" - }, - "tf-playground-svpc-net": { - "number": "1079408472053", - "project_id": "tf-playground-svpc-net" - }, - "tf-playground-svpc-gke": { - "number": "1043465648801", - "project_id": "tf-playground-svpc-gke" - } - }, - "projects:number": { - "64297462517": { - "number": "64297462517", - "project_id": "tf-playground-simple" - }, - "759592912116": { - "number": "759592912116", - "project_id": "ludo-dev-net-spoke-0" - }, - "233262889141": { - "number": "233262889141", - "project_id": "ludo-prod-net-landing-0" - }, - "195159130008": { - "number": "195159130008", - "project_id": "ludo-prod-net-spoke-0" - }, - "971935141727": { - "number": "971935141727", - "project_id": "ludo-dev-sec-core-0" - }, - "990827422843": { - "number": "990827422843", - "project_id": "ludo-prod-sec-core-0" - }, - "833078410166": { - "number": "833078410166", - "project_id": "tf-playground-gcs-test-0" - }, - "433746214138": { - "number": "433746214138", - "project_id": "tf-playground-svpc-gce-dr" - }, - "697669426824": { - "number": "697669426824", - "project_id": "tf-playground-svpc-net-dr" - }, - "515444627958": { - "number": "515444627958", - "project_id": "tf-playground-svpc-openshift" - }, - "783093469136": { - "number": "783093469136", - "project_id": "tf-playground-svpc-gce" - }, - "1079408472053": { - "number": "1079408472053", - "project_id": "tf-playground-svpc-net" - }, - "1043465648801": { - "number": "1043465648801", - "project_id": "tf-playground-svpc-gke" - } - }, "quota": { "tf-playground-simple": { "global": { @@ -2721,12 +2721,10 @@ ] }, "metric-descriptors": { - "custom.googleapis.com/netmon/firewall_policies/tuples_available": {}, - "custom.googleapis.com/netmon/firewall_policies/tuples_used": {}, - "custom.googleapis.com/netmon/firewall_policies/tuples_used_ratio": {}, - "custom.googleapis.com/netmon/network/firewall_rules_available": {}, + "custom.googleapis.com/netmon/firewall_policy/tuples_available": {}, + "custom.googleapis.com/netmon/firewall_policy/tuples_used": {}, + "custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio": {}, "custom.googleapis.com/netmon/network/firewall_rules_used": {}, - "custom.googleapis.com/netmon/network/firewall_rules_used_ratio": {}, "custom.googleapis.com/netmon/network/forwarding_rules_l4_available": {}, "custom.googleapis.com/netmon/network/forwarding_rules_l4_used": {}, "custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio": {}, @@ -2770,6 +2768,9 @@ "custom.googleapis.com/netmon/project/routes_dynamic_available": {}, "custom.googleapis.com/netmon/project/routes_dynamic_used": {}, "custom.googleapis.com/netmon/project/routes_dynamic_used_ratio": {}, + "custom.googleapis.com/netmon/project/routes_static_available": {}, + "custom.googleapis.com/netmon/project/routes_static_used": {}, + "custom.googleapis.com/netmon/project/routes_static_used_ratio": {}, "custom.googleapis.com/netmon/subnetwork/addresses_available": {}, "custom.googleapis.com/netmon/subnetwork/addresses_used": {}, "custom.googleapis.com/netmon/subnetwork/addresses_used_ratio": {} diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py index 09c1dbd0ac..1bdc4cb20b 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py @@ -39,7 +39,8 @@ defaults=[True]) Level = enum.IntEnum('Level', 'CORE PRIMARY DERIVED') MetricDescriptor = collections.namedtuple('MetricDescriptor', - 'type name labels is_ratio') + 'type name labels is_ratio', + defaults=[False]) Plugin = collections.namedtuple('Plugin', 'func name level priority', defaults=[Level.PRIMARY, 99]) Resource = collections.namedtuple('Resource', 'type id data key', diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py index 4c5fa3ca95..9c9e8f9486 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py @@ -11,11 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Discovers project quota via Compute API and overlay user overrides. + +This plugin discovers project quota via batch Compute API requests. Project and +network quotas are then optionally overlaid with custom quota modifiers passed +in as options. Region quota discovery is partially implemented but not active. +''' import logging from . import Level, Resource, register_init, register_discovery -from .utils import batched, dirty_mp_request, dirty_mp_response +from .utils import batched, poor_man_mp_request, poor_man_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-quota') NAME = 'quota' @@ -25,10 +31,12 @@ def _handle_discovery(resources, response): + 'Processes asset batch response and overlays custom quota.' LOGGER.info('discovery handle request') content_type = response.headers['content-type'] per_project_quota = resources['config:custom_quota'].get('projects', {}) - for part in dirty_mp_response(content_type, response.content): + # process batch response + for part in poor_man_mp_response(content_type, response.content): kind = part.get('kind') quota = { q['metric']: int(q['limit']) @@ -54,14 +62,14 @@ def _handle_discovery(resources, response): @register_init def init(resources): - 'Create the quota key in the shared resource map.' + 'Prepares quota datastructures in the shared resource map.' LOGGER.info('init') resources.setdefault(NAME, {}) @register_discovery(Level.DERIVED, 0) def start_discovery(resources, response=None): - 'Fetch and process quota for projects.' + 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') if response is None: # TODO: regions @@ -69,7 +77,7 @@ def start_discovery(resources, response=None): if not urls: return for batch in batched(urls, 10): - yield dirty_mp_request(batch) + yield poor_man_mp_request(batch) else: for result in _handle_discovery(resources, response): yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py index b40e05f0d6..cd2840b771 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py @@ -11,11 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Discovers dynamic route counts via router status. + +This plugin depends on the CAI Compute one as it discovers dynamic route +data by parsing router status, and it needs routers to have already been +discovered. It uses batch Compute API requests via the utils functions. +''' import logging from . import Level, Resource, register_init, register_discovery -from .utils import batched, dirty_mp_request, dirty_mp_response +from .utils import batched, poor_man_mp_request, poor_man_mp_response LOGGER = logging.getLogger('net-dash.discovery.compute-routes-dynamic') NAME = 'routes_dynamic' @@ -24,10 +30,13 @@ def _handle_discovery(resources, response): + 'Processes asset batch response and parses router status data.' LOGGER.info('discovery handle request') content_type = response.headers['content-type'] routers = [r for r in resources['routers'].values()] - for i, part in enumerate(dirty_mp_response(content_type, response.content)): + # process batch response + for i, part in enumerate(poor_man_mp_response(content_type, + response.content)): router = routers[i] result = part.get('result') if not result: @@ -57,12 +66,14 @@ def _handle_discovery(resources, response): @register_init def init(resources): + 'Prepares dynamic routes datastructure in the shared resource map.' LOGGER.info('init') resources.setdefault(NAME, {}) @register_discovery(Level.DERIVED) def start_discovery(resources, response=None): + 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') if not response: urls = [ @@ -72,7 +83,7 @@ def start_discovery(resources, response=None): if not urls: return for batch in batched(urls, 10): - yield dirty_mp_request(batch) + yield poor_man_mp_request(batch) else: for result in _handle_discovery(resources, response): yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py index 1b5f12c89e..350c288b4c 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Group discovered networks by project.' import itertools import logging @@ -23,12 +24,14 @@ @register_init def init(resources): + 'Prepares datastructure in the shared resource map.' LOGGER.info('init') resources.setdefault(NAME, {}) @register_discovery(Level.DERIVED) def start_discovery(resources, response=None): + 'Plugin entry point, group and return discovered networks.' LOGGER.info(f'discovery (has response: {response is not None})') grouped = itertools.groupby(resources['networks'].values(), lambda v: v['project_id']) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py index 509519bda7..3af926be6e 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py @@ -11,6 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Discover existing network dashboard metric descriptors. + +Populating this data allows the tool to later compute which metric descriptors +need to be created. +''' import logging import urllib.parse @@ -28,6 +33,7 @@ def _handle_discovery(resources, response, data): + 'Processes monitoring API response and parses descriptor data.' LOGGER.info('discovery handle request') descriptors = data.get('metricDescriptors') if not descriptors: @@ -43,12 +49,14 @@ def _handle_discovery(resources, response, data): @register_init def init(resources): + 'Prepares datastructure in the shared resource map.' LOGGER.info('init') resources.setdefault(NAME, {}) @register_discovery(Level.CORE, 0) def start_discovery(resources, response=None, data=None): + 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') project_id = resources['config:monitoring_project'] type_root = resources['config:monitoring_root'] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py index 922a244335..de4eae8971 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Utility functions to create monitoring API requests.' import collections import datetime @@ -30,6 +31,7 @@ def descriptor_requests(project_id, root, existing, computed): + 'Returns create requests for missing descriptors.' type_base = DESCRIPTOR_TYPE_BASE.format(root) url = DESCRIPTOR_URL.format(project_id) for descriptor in computed: @@ -59,10 +61,13 @@ def descriptor_requests(project_id, root, existing, computed): def timeseries_requests(project_id, root, timeseries, descriptors): + 'Returns create requests for timeseries.' descriptor_valuetypes = {d.type: d.is_ratio for d in descriptors} end_time = ''.join((datetime.datetime.utcnow().isoformat('T'), 'Z')) type_base = DESCRIPTOR_TYPE_BASE.format(root) url = TIMESERIES_URL.format(project_id) + # group timeseries in buckets by their type so that multiple timeseries + # can be grouped in a single API request without grouping duplicates types ts_buckets = {} for ts in timeseries: bucket = ts_buckets.setdefault(ts.metric, collections.deque()) @@ -98,4 +103,4 @@ def timeseries_requests(project_id, root, timeseries, descriptors): tot_num = sum(len(b) for b in ts_buckets) LOGGER.info(f'sending {req_num} remaining: {tot_num}') yield HTTPRequest(url, HEADERS, json.dumps(data)) - ts_buckets = [b for b in ts_buckets if b] \ No newline at end of file + ts_buckets = [b for b in ts_buckets if b] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py index e2dc0f99c5..defd697532 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Prepares descriptors and timeseries for firewall policy resources.' import logging from . import MetricDescriptor, TimeSeries, register_timeseries -DESCRIPTOR_PREFIX = 'firewall_policies' DESCRIPTOR_ATTRS = { 'tuples_used': 'Firewall tuples used per policy', 'tuples_available': 'Firewall tuples limit per policy', @@ -29,15 +29,15 @@ @register_timeseries def timeseries(resources): - 'Derive network timeseries for firewall policies.' + 'Returns used/available/ratio firewall tuples timeseries by policy.' LOGGER.info('timeseries') for dtype, name in DESCRIPTOR_ATTRS.items(): - yield MetricDescriptor(f'{DESCRIPTOR_PREFIX}/{dtype}', name, - DESCRIPTOR_LABELS, dtype.endswith('ratio')) + yield MetricDescriptor(f'firewall_policy/{dtype}', name, DESCRIPTOR_LABELS, + dtype.endswith('ratio')) for v in resources['firewall_policies'].values(): tuples = int(v['num_tuples']) labels = {'parent': v['parent'], 'name': v['name']} - yield TimeSeries('firewall_policies/tuples_used', tuples, labels) - yield TimeSeries('firewall_policies/tuples_available', TUPLE_LIMIT, labels) - yield TimeSeries('firewall_policies/tuples_used_ratio', - tuples / TUPLE_LIMIT, labels) + yield TimeSeries('firewall_policy/tuples_used', tuples, labels) + yield TimeSeries('firewall_policy/tuples_available', TUPLE_LIMIT, labels) + yield TimeSeries('firewall_policy/tuples_used_ratio', tuples / TUPLE_LIMIT, + labels) diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py index 67634ec306..5490e6d3bd 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Prepares descriptors and timeseries for firewall rules by project and network.' import itertools import logging @@ -18,22 +19,25 @@ from . import MetricDescriptor, TimeSeries, register_timeseries DESCRIPTOR_ATTRS = { - 'firewall_rules_used': 'Firewall rules used per {}', - 'firewall_rules_available': 'Firewall rules limit per {}', - 'firewall_rules_used_ratio': 'Firewall rules used ratio per {}', + 'firewall_rules_used': 'Firewall rules used per project', + 'firewall_rules_available': 'Firewall rules limit per project', + 'firewall_rules_used_ratio': 'Firewall rules used ratio per project', } LOGGER = logging.getLogger('net-dash.timeseries.firewall-rules') @register_timeseries def timeseries(resources): - 'Derive and yield network and project timeseries for firewall rules.' + 'Returns used/available/ratio firewall timeseries by project and network.' LOGGER.info('timeseries') - for prefix in ('network', 'project'): - for dtype, name in DESCRIPTOR_ATTRS.items(): - labels = ('project',) if prefix == 'project' else ('project', 'name') - yield MetricDescriptor(f'{prefix}/{dtype}', name.format(prefix), labels, - dtype.endswith('ratio')) + # return a single descriptor for network as we don't have limits + yield MetricDescriptor(f'network/firewall_rules_used', + 'Firewall rules used per network', ('project', 'name')) + # return used/vailable/ratio descriptors for project + for dtype, name in DESCRIPTOR_ATTRS.items(): + yield MetricDescriptor(f'project/{dtype}', name, ('project',), + dtype.endswith('ratio')) + # group firewall rules by network then prepare and return timeseries grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['network']) for network_id, rules in grouped: @@ -43,6 +47,7 @@ def timeseries(resources): 'project': resources['networks'][network_id]['project_id'] } yield TimeSeries('network/firewall_rules_used', count, labels) + # group firewall rules by project then prepare and return timeseries grouped = itertools.groupby(resources['firewall_rules'].values(), lambda v: v['project_id']) for project_id, rules in grouped: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py index ef66723c98..0ce7a4b304 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py @@ -11,11 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'''Prepares descriptors and timeseries for network-level metrics. + +This plugin computes metrics for a variety of network resource types like +subnets, instances, peerings, etc. It mostly does so by first grouping +resources for a type, and then using a generalized function to derive counts +and ratios and compute the actual timeseries. +''' import functools import itertools import logging -import operator from . import MetricDescriptor, TimeSeries, register_timeseries @@ -50,7 +56,7 @@ def _group_timeseries(name, resources, grouped, limit_name): - 'Derive and yield timeseries from grouped iterators and limits.' + 'Generalized function that returns timeseries from data grouped by network.' for network_id, elements in grouped: network = resources['networks'].get(network_id) if not network: @@ -66,13 +72,15 @@ def _group_timeseries(name, resources, grouped, limit_name): def _forwarding_rules(resources): - 'Derive network timeseries for forwarding rule utilization.' + 'Groups forwarding rules by network/type and returns relevant timeseries.' + # create two separate iterators filtered by L4 and L7 balancing schemes filter = lambda n, v: v['load_balancing_scheme'] != n forwarding_rules = resources['forwarding_rules'].values() forwarding_rules_l4 = itertools.filterfalse( functools.partial(filter, 'INTERNAL'), forwarding_rules) forwarding_rules_l7 = itertools.filterfalse( functools.partial(filter, 'INTERNAL_MANAGED'), forwarding_rules) + # group each iterator by network and return timeseries grouped_l4 = itertools.groupby(forwarding_rules_l4, lambda i: i['network']) grouped_l7 = itertools.groupby(forwarding_rules_l7, lambda i: i['network']) return itertools.chain( @@ -84,7 +92,7 @@ def _forwarding_rules(resources): def _instances(resources): - 'Derive network timeseries for instance utilization.' + 'Groups instances by network and returns relevant timeseries.' instance_networks = itertools.chain.from_iterable( i['networks'] for i in resources['instances'].values()) grouped = itertools.groupby(instance_networks, lambda i: i['network']) @@ -93,6 +101,7 @@ def _instances(resources): def _peerings(resources): + 'Counts peerings by network and returns relevant timeseries.' quota = resources['quota'] for network_id, network in resources['networks'].items(): labels = {'project': network['project_id'], 'network': network['name']} @@ -110,7 +119,7 @@ def _peerings(resources): def _subnet_ranges(resources): - 'Derive network timeseries for subnet range utilization.' + 'Groups subnetworks by network and returns relevant timeseries.' grouped = itertools.groupby(resources['subnetworks'].values(), lambda v: v['network']) return _group_timeseries('subnets', resources, grouped, @@ -119,11 +128,14 @@ def _subnet_ranges(resources): @register_timeseries def timeseries(resources): - 'Yield timeseries.' + 'Returns used/available/ratio timeseries by network for different resources.' LOGGER.info('timeseries') + # return descriptors for dtype, name in DESCRIPTOR_ATTRS.items(): yield MetricDescriptor(f'network/{dtype}', name, ('project', 'network'), dtype.endswith('ratio')) + + # chain iterators from specialized functions and yield combined timeseries results = itertools.chain(_forwarding_rules(resources), _instances(resources), _peerings(resources), _subnet_ranges(resources)) for result in results: diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py index 4778b00f45..9f79268500 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Prepares descriptors and timeseries for peering group metrics.' import itertools import logging @@ -75,6 +76,7 @@ def _count_forwarding_rules_l4(resources, network_ids): + 'Returns count of L4 forwarding rules for specified network ids.' return len([ r for r in resources['forwarding_rules'].values() if r['network'] in network_ids and r['load_balancing_scheme'] == 'INTERNAL' @@ -82,6 +84,7 @@ def _count_forwarding_rules_l4(resources, network_ids): def _count_forwarding_rules_l7(resources, network_ids): + 'Returns count of L7 forwarding rules for specified network ids.' return len([ r for r in resources['forwarding_rules'].values() if r['network'] in network_ids and @@ -90,6 +93,7 @@ def _count_forwarding_rules_l7(resources, network_ids): def _count_instances(resources, network_ids): + 'Returns count of instances for specified network ids.' count = 0 for i in resources['instances'].values(): if any(n['network'] in network_ids for n in i['networks']): @@ -98,11 +102,13 @@ def _count_instances(resources, network_ids): def _count_routes_static(resources, network_ids): + 'Returns count of static routes for specified network ids.' return len( [r for r in resources['routes'].values() if r['network'] in network_ids]) def _count_routes_dynamic(resources, network_ids): + 'Returns count of dynamic routes for specified network ids.' return sum([ sum(v.values()) for k, v in resources['routes_dynamic'].items() @@ -111,6 +117,7 @@ def _count_routes_dynamic(resources, network_ids): def _get_limit_max(quota, network_id, project_id, resource_name): + 'Returns maximum limit value in project / peering group / network limits.' pg_name, pg_default = LIMITS[resource_name]['pg'] prj_name, prj_default = LIMITS[resource_name]['prj'] network_quota = quota.get(network_id, {}) @@ -123,21 +130,23 @@ def _get_limit_max(quota, network_id, project_id, resource_name): def _get_limit(quota, network, resource_name): - # https://cloud.google.com/vpc/docs/quota#vpc-peering-ilb-example - # vpc_max = max(vpc limit, pg limit) + 'Computes and returns peering group limit.' + # reference https://cloud.google.com/vpc/docs/quota#vpc-peering-ilb-example + # step 1 - vpc_max = max(vpc limit, pg limit) vpc_max = _get_limit_max(quota, network['self_link'], network['project_id'], resource_name) - # peers_max = [max(vpc limit, pg limit) for v in peered vpcs] - # peers_min = min(peers_max) + # step 2 - peers_max = [max(vpc limit, pg limit) for v in peered vpcs] + # step 3 - peers_min = min(peers_max) peers_min = min([ _get_limit_max(quota, p['network'], p['project_id'], resource_name) for p in network['peerings'] ]) - # max(vpc_max, peers_min) + # step 4 - max(vpc_max, peers_min) return max([vpc_max, peers_min]) -def _network_timeseries(resources, network): +def _peering_group_timeseries(resources, network): + 'Computes and returns peering group timeseries for network.' if len(network['peerings']) == 0: return network_ids = [network['self_link'] @@ -158,12 +167,14 @@ def _network_timeseries(resources, network): @register_timeseries def timeseries(resources): - 'Yield timeseries.' + 'Returns peering group timeseries for all networks.' LOGGER.info('timeseries') + # returns metric descriptors for dtype, name in DESCRIPTOR_ATTRS.items(): yield MetricDescriptor(f'peering_group/{dtype}', name, ('project', 'network'), dtype.endswith('ratio')) - results = itertools.chain(*(_network_timeseries(resources, n) + # chain timeseries for each network and return each one individually + results = itertools.chain(*(_peering_group_timeseries(resources, n) for n in resources['networks'].values())) for result in results: yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py index 83764d024f..89011215ca 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Prepares descriptors and timeseries for network-level route metrics.' import itertools import logging @@ -44,7 +45,7 @@ def _dynamic(resources): - 'Derive network timeseries for dynamic routes.' + 'Computes network-level timeseries for dynamic routes.' for network_id, router_counts in resources['routes_dynamic'].items(): network = resources['networks'][network_id] count = sum(router_counts.values()) @@ -56,7 +57,7 @@ def _dynamic(resources): def _static(resources): - 'Derive network and project timeseries for static routes.' + 'Computes network and project-level timeseries for dynamic routes.' filter = lambda v: v['next_hop_type'] in ('peering', 'network') routes = itertools.filterfalse(filter, resources['routes'].values()) grouped = itertools.groupby(routes, lambda v: v['network']) @@ -79,12 +80,14 @@ def _static(resources): @register_timeseries def timeseries(resources): - 'Yield timeseries.' + 'Returns used/available/ratio timeseries by network and project.' LOGGER.info('timeseries') + # return descriptors for dtype, name in DESCRIPTOR_ATTRS.items(): labels = ('project') if dtype.startswith('project') else ('project', 'network') yield MetricDescriptor(dtype, name, labels, dtype.endswith('ratio')) + # chain static and dynamic route timeseries then return each one individually results = itertools.chain(_static(resources), _dynamic(resources)) for result in results: yield result diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py index b2cc6b4811..a9f0a5f302 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Prepares descriptors and timeseries for subnetwork-level metrics.' import collections import ipaddress @@ -28,7 +29,7 @@ def _subnet_addresses(resources): - 'Return partial counts of addresses per subnetwork.' + 'Returns count of addresses per subnetwork.' for v in resources['addresses'].values(): if v['status'] != 'RESERVED': continue @@ -37,7 +38,7 @@ def _subnet_addresses(resources): def _subnet_forwarding_rules(resources, subnet_nets): - 'Return partial counts of forwarding rules per subnetwork.' + 'Returns counts of forwarding rules per subnetwork.' for v in resources['forwarding_rules'].values(): if v['load_balancing_scheme'].startswith('INTERNAL'): yield v['subnetwork'], 1 @@ -56,7 +57,7 @@ def _subnet_forwarding_rules(resources, subnet_nets): def _subnet_instances(resources): - 'Return partial counts of instances per subnetwork.' + 'Returns counts of instances per subnetwork.' vm_networks = itertools.chain.from_iterable( i['networks'] for i in resources['instances'].values()) return collections.Counter(v['subnetwork'] for v in vm_networks).items() @@ -64,23 +65,26 @@ def _subnet_instances(resources): @register_timeseries def timeseries(resources): - 'Derive and yield subnetwork timeseries for address utilization.' + 'Returns used/available/ratio timeseries for addresses by subnetwork.' LOGGER.info('timeseries') + # return descriptors for dtype, name in DESCRIPTOR_ATTRS.items(): yield MetricDescriptor(f'subnetwork/{dtype}', name, ('project', 'network', 'subnetwork', 'region'), dtype.endswith('ratio')) + # aggregate per-resource counts in total per-subnet counts subnet_nets = { k: ipaddress.ip_network(v['cidr_range']) for k, v in resources['subnetworks'].items() } + # TODO: add counter functions for PSA subnet_counts = {k: 0 for k in resources['subnetworks']} - # TODO: PSA counters = itertools.chain(_subnet_addresses(resources), _subnet_forwarding_rules(resources, subnet_nets), _subnet_instances(resources)) for subnet_self_link, count in counters: subnet_counts[subnet_self_link] += count + # compute and return metrics for subnet_self_link, count in subnet_counts.items(): max_ips = subnet_nets[subnet_self_link].num_addresses - 4 subnet = resources['subnetworks'][subnet_self_link] diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py index eccce24b14..5be6599889 100644 --- a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py +++ b/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Utility functions for API requests and responses.' import itertools import json @@ -37,7 +38,7 @@ def batched(iterable, n): - 'Batch data into lists of length n. The last batch may be shorter.' + 'Batches data into lists of length n. The last batch may be shorter.' # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: raise ValueError('n must be at least one') @@ -46,8 +47,30 @@ def batched(iterable, n): yield batch -def dirty_mp_request(urls, boundary='1234567890'): - 'Bundle urls into a single multipart mixed batched request.' +def parse_cai_results(data, name, resource_type=None, method='search'): + 'Parses an asset API response and returns individual results.' + results = data.get('results' if method == 'search' else 'assets') + if not results: + logging.info(f'no results for {name}') + return + for result in results: + if resource_type and result['assetType'] != resource_type: + logging.warn(f'result for wrong type {result["assetType"]}') + continue + yield result + + +def parse_page_token(data, url): + 'Detect next page token in result and return next page URL.' + page_token = data.get('nextPageToken') + if page_token: + logging.info(f'page token {page_token}') + if page_token: + return RE_URL.sub(f'pageToken={page_token}&', url) + + +def poor_man_mp_request(urls, boundary='1234567890'): + 'Bundles URLs into a single multipart mixed batched request.' boundary = f'--{boundary}' data = [boundary] for url in urls: @@ -58,8 +81,8 @@ def dirty_mp_request(urls, boundary='1234567890'): ''.join(data), False) -def dirty_mp_response(content_type, content): - 'Parse multipart mixed response and return individual parts.' +def poor_man_mp_response(content_type, content): + 'Parses a multipart mixed response and returns individual parts.' try: _, boundary = content_type.split('=') except ValueError: @@ -76,25 +99,3 @@ def dirty_mp_response(content_type, content): except ValueError: raise PluginError('cannot parse MIME part') yield json.loads(body) - - -def parse_cai_results(data, name, resource_type=None, method='search'): - 'Preliminary parsing of CAI asset result.' - results = data.get('results' if method == 'search' else 'assets') - if not results: - logging.info(f'no results for {name}') - return - for result in results: - if resource_type and result['assetType'] != resource_type: - logging.warn(f'result for wrong type {result["assetType"]}') - continue - yield result - - -def parse_page_token(data, url): - 'Detect next page token in result and return next page URL.' - page_token = data.get('nextPageToken') - if page_token: - logging.info(f'page token {page_token}') - if page_token: - return RE_URL.sub(f'pageToken={page_token}&', url) diff --git a/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py b/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py index 5f639c302e..c0645d91cb 100755 --- a/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py +++ b/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +'Delete metric descriptors matching filter.' import json import logging @@ -51,6 +52,7 @@ def fetch(url, delete=False): @click.option('--op-project', '-op', required=True, type=str, help='GCP monitoring project where metrics will be stored.') def main(op_project): + 'Module entry point.' # if not click.confirm('Do you want to continue?'): # raise SystemExit(0) logging.info('fetching descriptors') From a721ab96feca43fe462fb9b62c905304cbd391f2 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 03:39:16 +0100 Subject: [PATCH 62/82] rename cf to src --- .../cloud-operations/network-dashboard/{cf => src}/MULTIPART.md | 0 .../cloud-operations/network-dashboard/{cf => src}/NOTES.md | 0 .../network-dashboard/{cf => src}/custom-quotas.sample | 0 blueprints/cloud-operations/network-dashboard/{cf => src}/main.py | 0 .../cloud-operations/network-dashboard/{cf => src}/out.json | 0 .../network-dashboard/{cf => src}/plugins/__init__.py | 0 .../{cf => src}/plugins/core-discover-cai-projects.py | 0 .../network-dashboard/{cf => src}/plugins/discover-cai-compute.py | 0 .../{cf => src}/plugins/discover-compute-quota.py | 0 .../{cf => src}/plugins/discover-compute-routerstatus.py | 0 .../{cf => src}/plugins/discover-group-networks.py | 0 .../{cf => src}/plugins/discover-metric-descriptors.py | 0 .../network-dashboard/{cf => src}/plugins/monitoring.py | 0 .../{cf => src}/plugins/series-firewall-policies.py | 0 .../{cf => src}/plugins/series-firewall-rules.py | 0 .../network-dashboard/{cf => src}/plugins/series-networks.py | 0 .../{cf => src}/plugins/series-peering-groups.py | 0 .../network-dashboard/{cf => src}/plugins/series-routes.py | 0 .../network-dashboard/{cf => src}/plugins/series-subnets.py | 0 .../network-dashboard/{cf => src}/plugins/utils.py | 0 .../network-dashboard/{cf => src}/requirements.txt | 0 .../network-dashboard/{cf => src}/tools/remove-descriptors.py | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename blueprints/cloud-operations/network-dashboard/{cf => src}/MULTIPART.md (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/NOTES.md (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/custom-quotas.sample (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/main.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/out.json (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/__init__.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/core-discover-cai-projects.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/discover-cai-compute.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/discover-compute-quota.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/discover-compute-routerstatus.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/discover-group-networks.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/discover-metric-descriptors.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/monitoring.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/series-firewall-policies.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/series-firewall-rules.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/series-networks.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/series-peering-groups.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/series-routes.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/series-subnets.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/plugins/utils.py (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/requirements.txt (100%) rename blueprints/cloud-operations/network-dashboard/{cf => src}/tools/remove-descriptors.py (100%) diff --git a/blueprints/cloud-operations/network-dashboard/cf/MULTIPART.md b/blueprints/cloud-operations/network-dashboard/src/MULTIPART.md similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/MULTIPART.md rename to blueprints/cloud-operations/network-dashboard/src/MULTIPART.md diff --git a/blueprints/cloud-operations/network-dashboard/cf/NOTES.md b/blueprints/cloud-operations/network-dashboard/src/NOTES.md similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/NOTES.md rename to blueprints/cloud-operations/network-dashboard/src/NOTES.md diff --git a/blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample b/blueprints/cloud-operations/network-dashboard/src/custom-quotas.sample similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/custom-quotas.sample rename to blueprints/cloud-operations/network-dashboard/src/custom-quotas.sample diff --git a/blueprints/cloud-operations/network-dashboard/cf/main.py b/blueprints/cloud-operations/network-dashboard/src/main.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/main.py rename to blueprints/cloud-operations/network-dashboard/src/main.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/out.json b/blueprints/cloud-operations/network-dashboard/src/out.json similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/out.json rename to blueprints/cloud-operations/network-dashboard/src/out.json diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py b/blueprints/cloud-operations/network-dashboard/src/plugins/__init__.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/__init__.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/__init__.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/core-discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-projects.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/core-discover-cai-projects.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-projects.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-cai-compute.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-compute-quota.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-quota.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/discover-compute-quota.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-compute-routerstatus.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-compute-routerstatus.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/discover-compute-routerstatus.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-group-networks.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-group-networks.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/discover-group-networks.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-metric-descriptors.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/discover-metric-descriptors.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/discover-metric-descriptors.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py b/blueprints/cloud-operations/network-dashboard/src/plugins/monitoring.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/monitoring.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/monitoring.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-policies.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-policies.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-policies.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-rules.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-firewall-rules.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/series-firewall-rules.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-networks.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-networks.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/series-networks.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-peering-groups.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-peering-groups.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/series-peering-groups.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-routes.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-routes.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/series-routes.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-subnets.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/series-subnets.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/series-subnets.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py b/blueprints/cloud-operations/network-dashboard/src/plugins/utils.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/plugins/utils.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/utils.py diff --git a/blueprints/cloud-operations/network-dashboard/cf/requirements.txt b/blueprints/cloud-operations/network-dashboard/src/requirements.txt similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/requirements.txt rename to blueprints/cloud-operations/network-dashboard/src/requirements.txt diff --git a/blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py b/blueprints/cloud-operations/network-dashboard/src/tools/remove-descriptors.py similarity index 100% rename from blueprints/cloud-operations/network-dashboard/cf/tools/remove-descriptors.py rename to blueprints/cloud-operations/network-dashboard/src/tools/remove-descriptors.py From a64f7542d91ea58b66f0ce22c6e17b35c87432e2 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 04:06:11 +0100 Subject: [PATCH 63/82] discover nodes instead of just projects --- ...projects.py => core-discover-cai-nodes.py} | 47 ++++++++++--------- .../src/plugins/discover-cai-compute.py | 2 +- 2 files changed, 27 insertions(+), 22 deletions(-) rename blueprints/cloud-operations/network-dashboard/src/plugins/{core-discover-cai-projects.py => core-discover-cai-nodes.py} (61%) diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-projects.py b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py similarity index 61% rename from blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-projects.py rename to blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py index 5705ded99c..2eb2f5b19d 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-projects.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py @@ -11,12 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -'''Project discovery from configuration options. +'''Project and folder discovery from configuration options. -This plugin needs to run first, as it's responsible for discovering projects -and their attributes, based on configuration options. Projects are fetched -from Cloud Asset Inventory based on explicit id or being part of a folder -hierarchy. +This plugin needs to run first, as it's responsible for discovering nodes that +contain resources: folders and projects contained in the hierarchy passed in +via configuration options. Node resources are fetched from Cloud Asset +Inventory based on explicit id or being part of a folder hierarchy. ''' import logging @@ -25,25 +25,27 @@ from .utils import parse_page_token, parse_cai_results LOGGER = logging.getLogger('net-dash.discovery.cai-projects') -NAME = 'projects' -TYPE = 'cloudresourcemanager.googleapis.com/Project' -CAI_URL = ( - 'https://content-cloudasset.googleapis.com/v1p1beta1' - '/{}/resources:searchAll' - '?assetTypes=cloudresourcemanager.googleapis.com/Project&pageSize=500') +CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' + '/{}/resources:searchAll' + '?assetTypes=cloudresourcemanager.googleapis.com/Folder' + '&assetTypes=cloudresourcemanager.googleapis.com/Project' + '&pageSize=500') def _handle_discovery(resources, response, data): 'Processes asset response and returns project resources or next URLs.' LOGGER.info('discovery handle request') - for result in parse_cai_results(data, NAME, TYPE): - data = { - 'number': result['project'].split('/')[1], - 'project_id': result['displayName'] - } - yield Resource('projects', data['project_id'], data) - yield Resource('projects:number', data['number'], data) + for result in parse_cai_results(data, 'nodes'): + asset_type = result['assetType'].split('/')[-1] + name = result['name'].split('/')[-1] + if asset_type == 'Folder': + yield Resource('folders', name, {'name': result['displayName']}) + elif asset_type == 'Project': + number = result['project'].split('/')[1] + data = {'number': number, 'project_id': name} + yield Resource('projects', name, data) + yield Resource('projects:number', number, data) next_url = parse_page_token(data, response.request.url) if next_url: LOGGER.info('discovery next url') @@ -54,7 +56,8 @@ def _handle_discovery(resources, response, data): def init(resources): 'Prepares project datastructures in the shared resource map.' LOGGER.info('init') - resources.setdefault(NAME, {}) + resources.setdefault('folders', {}) + resources.setdefault('projects', {}) resources.setdefault('projects:number', {}) @@ -64,10 +67,12 @@ def start_discovery(resources, response=None, data=None): LOGGER.info(f'discovery (has response: {response is not None})') if response is None: # return asset discovery URLs from initial options on first call - for v in resources['config:projects']: - yield HTTPRequest(CAI_URL.format(f'projects/{v}'), {}, None) for v in resources['config:folders']: + yield Resource('folders', v.split('/')[-1], {}) yield HTTPRequest(CAI_URL.format(f'folders/{v}'), {}, None) + for v in resources['config:projects']: + if v not in resources['projects']: + yield HTTPRequest(CAI_URL.format(f'projects/{v}'), {}, None) else: # pass the API response to the plugin data handler and return results for result in _handle_discovery(resources, response, data): diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py index 59283673c3..1b471af910 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py @@ -194,7 +194,7 @@ def _get_parent(parent, resources): if project: return {'project_id': project['project_id'], 'project_number': parent_id} if parent_type == 'folders': - if parent_id in resources['config:folders']: + if parent_id in resources['folders']: return {'parent': f'{parent_type}/{parent_id}'} if resources['config:organization'] == int(parent_id): return {'parent': f'{parent_type}/{parent_id}'} From a0b085a53ef06cc328ba44ec47ff5f2c8fdecef3 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 04:25:31 +0100 Subject: [PATCH 64/82] discovery node can be a folder or org --- .../network-dashboard/src/main.py | 26 ++++--- .../network-dashboard/src/out.json | 72 ++++++++++++------- .../src/plugins/core-discover-cai-nodes.py | 5 +- .../src/plugins/discover-cai-compute.py | 8 +-- 4 files changed, 69 insertions(+), 42 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/src/main.py b/blueprints/cloud-operations/network-dashboard/src/main.py index 7da320c2eb..39c7f693ba 100755 --- a/blueprints/cloud-operations/network-dashboard/src/main.py +++ b/blueprints/cloud-operations/network-dashboard/src/main.py @@ -78,23 +78,28 @@ def do_discovery(resources): {k: len(v) for k, v in resources.items() if not isinstance(v, str)})) -def do_init(resources, organization, op_project, folders=None, projects=None, +def do_init(resources, discovery_root, op_project, folders=None, projects=None, custom_quota=None): '''Calls init plugins to configure keys in the shared resource map. Args: - organization: organization id from configuration. + discovery_root: root node for discovery from configuration. op_project: monitoring project id id from configuration. folders: list of folder ids for resource discovery from configuration. projects: list of project ids for resource discovery from configuration. ''' LOGGER.info(f'init start') - resources['config:organization'] = str(organization) + folders = [str(f) for f in folders or []] + resources['config:discovery_root'] = discovery_root resources['config:monitoring_project'] = op_project - resources['config:folders'] = [str(f) for f in folders or []] + resources['config:folders'] = folders resources['config:projects'] = projects or [] resources['config:custom_quota'] = custom_quota or {} resources['config:monitoring_root'] = MONITORING_ROOT + if discovery_root.startswith('organization'): + resources['organization'] = discovery_root.split('/')[-1] + for f in folders: + resources['folders'] = {f: {} for f in folders} for plugin in plugins.get_init_plugins(): plugin.func(resources) @@ -201,8 +206,10 @@ def fetch(request): @click.command() -@click.option('--organization', '-o', required=True, type=int, - help='GCP organization id.') +@click.option( + '--discovery-root', '-dr', required=True, + help='Node used for asset discovery, either organizations/nnn or folders/nnn.' +) @click.option('--op-project', '-op', required=True, type=str, help='GCP monitoring project where metrics will be stored.') @click.option('--project', '-p', type=str, multiple=True, @@ -217,11 +224,13 @@ def fetch(request): help='Load JSON resources from file, skips init and discovery.') @click.option('--debug-plugin', help='Run only core and specified timeseries plugin.') -def main(organization=None, op_project=None, project=None, folder=None, +def main(discovery_root=None, op_project=None, project=None, folder=None, custom_quota_file=None, dump_file=None, load_file=None, debug_plugin=None): 'CLI entry point.' logging.basicConfig(level=logging.INFO) + if discovery_root.partition('/')[0] not in ('folders', 'organizations'): + raise SystemExit('Invalid discovery root.') descriptors = [] timeseries = [] if load_file: @@ -234,7 +243,8 @@ def main(organization=None, op_project=None, project=None, folder=None, custom_quota = yaml.load(custom_quota_file, Loader=yaml.Loader) except yaml.YAMLError as e: raise SystemExit(f'Error decoding custom quota file: {e.args[0]}') - do_init(resources, organization, op_project, folder, project, custom_quota) + do_init(resources, discovery_root, op_project, folder, project, + custom_quota) do_discovery(resources) if dump_file: json.dump(resources, dump_file, indent=2) diff --git a/blueprints/cloud-operations/network-dashboard/src/out.json b/blueprints/cloud-operations/network-dashboard/src/out.json index 52a533f263..134a8def6d 100644 --- a/blueprints/cloud-operations/network-dashboard/src/out.json +++ b/blueprints/cloud-operations/network-dashboard/src/out.json @@ -1,5 +1,5 @@ { - "config:organization": "436789450919", + "config:discovery_root": "organizations/436789450919", "config:monitoring_project": "tf-playground-svpc-net", "config:folders": [ "821058723541", @@ -24,11 +24,25 @@ } }, "config:monitoring_root": "netmon/", - "projects": { - "tf-playground-simple": { - "number": "64297462517", - "project_id": "tf-playground-simple" + "organization": "436789450919", + "folders": { + "821058723541": { + "name": "Networking" }, + "949988871993": { + "name": "Security" + }, + "321477570496": { + "name": "Sandbox" + }, + "1027067030104": { + "name": "Development" + }, + "948763653807": { + "name": "Production" + } + }, + "projects": { "ludo-dev-net-spoke-0": { "number": "759592912116", "project_id": "ludo-dev-net-spoke-0" @@ -76,13 +90,13 @@ "tf-playground-svpc-gke": { "number": "1043465648801", "project_id": "tf-playground-svpc-gke" + }, + "tf-playground-simple": { + "number": "64297462517", + "project_id": "tf-playground-simple" } }, "projects:number": { - "64297462517": { - "number": "64297462517", - "project_id": "tf-playground-simple" - }, "759592912116": { "number": "759592912116", "project_id": "ludo-dev-net-spoke-0" @@ -130,6 +144,10 @@ "1043465648801": { "number": "1043465648801", "project_id": "tf-playground-svpc-gke" + }, + "64297462517": { + "number": "64297462517", + "project_id": "tf-playground-simple" } }, "addresses": { @@ -2104,7 +2122,7 @@ } }, "quota": { - "tf-playground-simple": { + "ludo-dev-net-spoke-0": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2149,7 +2167,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "ludo-dev-net-spoke-0": { + "ludo-prod-net-landing-0": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2194,7 +2212,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "ludo-prod-net-landing-0": { + "ludo-prod-net-spoke-0": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2239,7 +2257,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "ludo-prod-net-spoke-0": { + "ludo-dev-sec-core-0": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2284,7 +2302,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "ludo-dev-sec-core-0": { + "ludo-prod-sec-core-0": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2329,7 +2347,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "ludo-prod-sec-core-0": { + "tf-playground-gcs-test-0": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2374,7 +2392,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "tf-playground-gcs-test-0": { + "tf-playground-svpc-gce-dr": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2419,7 +2437,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "tf-playground-svpc-gce-dr": { + "tf-playground-svpc-net-dr": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2464,7 +2482,7 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "tf-playground-svpc-net-dr": { + "tf-playground-svpc-openshift": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2509,10 +2527,10 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "tf-playground-svpc-openshift": { + "tf-playground-svpc-gce": { "global": { "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, + "BACKEND_SERVICES": 9, "EXTERNAL_VPN_GATEWAYS": 15, "FIREWALLS": 200, "FORWARDING_RULES": 45, @@ -2554,10 +2572,10 @@ "XPN_SERVICE_PROJECTS": 1000 } }, - "tf-playground-svpc-gce": { + "tf-playground-svpc-net": { "global": { "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 9, + "BACKEND_SERVICES": 75, "EXTERNAL_VPN_GATEWAYS": 15, "FIREWALLS": 200, "FORWARDING_RULES": 45, @@ -2596,10 +2614,11 @@ "URL_MAPS": 30, "VPN_GATEWAYS": 15, "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 + "XPN_SERVICE_PROJECTS": 1000, + "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 } }, - "tf-playground-svpc-net": { + "tf-playground-svpc-gke": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, @@ -2641,11 +2660,10 @@ "URL_MAPS": 30, "VPN_GATEWAYS": 15, "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000, - "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 + "XPN_SERVICE_PROJECTS": 1000 } }, - "tf-playground-svpc-gke": { + "tf-playground-simple": { "global": { "BACKEND_BUCKETS": 9, "BACKEND_SERVICES": 75, diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py index 2eb2f5b19d..94a5ed18fb 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py @@ -24,7 +24,7 @@ from . import HTTPRequest, Level, Resource, register_init, register_discovery from .utils import parse_page_token, parse_cai_results -LOGGER = logging.getLogger('net-dash.discovery.cai-projects') +LOGGER = logging.getLogger('net-dash.discovery.cai-nodes') CAI_URL = ('https://content-cloudasset.googleapis.com/v1p1beta1' '/{}/resources:searchAll' @@ -66,9 +66,8 @@ def start_discovery(resources, response=None, data=None): 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') if response is None: - # return asset discovery URLs from initial options on first call + # return initial discovery URLs for v in resources['config:folders']: - yield Resource('folders', v.split('/')[-1], {}) yield HTTPRequest(CAI_URL.format(f'folders/{v}'), {}, None) for v in resources['config:projects']: if v not in resources['projects']: diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py index 1b471af910..77269c000b 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py @@ -27,7 +27,7 @@ # https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network CAI_URL = ('https://content-cloudasset.googleapis.com/v1' - '/organizations/{organization}/assets' + '/{root}/assets' '?contentType=RESOURCE&{asset_types}&pageSize=500') LOGGER = logging.getLogger('net-dash.discovery.cai-compute') TYPES = { @@ -196,16 +196,16 @@ def _get_parent(parent, resources): if parent_type == 'folders': if parent_id in resources['folders']: return {'parent': f'{parent_type}/{parent_id}'} - if resources['config:organization'] == int(parent_id): + if resources.get('organization') == parent_id: return {'parent': f'{parent_type}/{parent_id}'} def _url(resources): 'Returns discovery URL' - organization = resources['config:organization'] + discovery_root = resources['config:discovery_root'] asset_types = '&'.join( f'assetTypes=compute.googleapis.com/{t}' for t in TYPES.values()) - return CAI_URL.format(organization=organization, asset_types=asset_types) + return CAI_URL.format(root=discovery_root, asset_types=asset_types) @register_init From 1dcbdb0dbbb90e1b0f8c695ba48b4487a3b4b208 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 08:29:05 +0100 Subject: [PATCH 65/82] cf entrypoint and fixes --- .../network-dashboard/src/MULTIPART.md | 44 - .../network-dashboard/src/NOTES.md | 70 - .../network-dashboard/src/main.py | 50 +- .../network-dashboard/src/out.json | 2796 ----------------- .../src/plugins/core-discover-cai-nodes.py | 2 + .../src/plugins/discover-cai-compute.py | 32 +- .../plugins/discover-metric-descriptors.py | 2 +- 7 files changed, 67 insertions(+), 2929 deletions(-) delete mode 100644 blueprints/cloud-operations/network-dashboard/src/MULTIPART.md delete mode 100644 blueprints/cloud-operations/network-dashboard/src/NOTES.md delete mode 100644 blueprints/cloud-operations/network-dashboard/src/out.json diff --git a/blueprints/cloud-operations/network-dashboard/src/MULTIPART.md b/blueprints/cloud-operations/network-dashboard/src/MULTIPART.md deleted file mode 100644 index be8a227cb5..0000000000 --- a/blueprints/cloud-operations/network-dashboard/src/MULTIPART.md +++ /dev/null @@ -1,44 +0,0 @@ -==== request start ==== -uri: -method: POST -== headers start == -b'authorization': --- Token Redacted --- -b'content-length': b'466' -b'content-type': b'multipart/mixed; boundary="===============2279194272988243142=="' -b'user-agent': b'google-cloud-sdk gcloud/409.0.0 command/gcloud.compute.project-info.describe invocation-id/ecaeec2337e24223910c5efe1c6ac176 environment/None environment-version/None interactive/True from-script/False python/3.9.12 term/xterm-256color (Linux 5.10.147-20147-gbf231eecc4e8)' -== headers end == -== body start == ---===============2279194272988243142== -Content-Type: application/http -MIME-Version: 1.0 -Content-Transfer-Encoding: binary -Content-ID: <8efd7e4e-8c7a-4d3c-9d75-85e156e4231d+0> - -GET /compute/v1/projects/tf-playground-svpc-net?alt=json HTTP/1.1 -Content-Type: application/json -MIME-Version: 1.0 -content-length: 0 -user-agent: google-cloud-sdk -accept: application/json -accept-encoding: gzip, deflate -Host: compute.googleapis.com - ---===============2279194272988243142==-- - -== body end == -==== request end ==== - ->>> from email.mime.multipart import MIMEMultipart, MIMEBase ->>> msg = MIMEMultipart("mixed") ->>> base = MIMEBase("application", "html") ->>> base.set_payload('''GET /compute/v1/projects/tf-playground-svpc-net?alt=json HTTP/1.1 -... Content-Type: application/json -... MIME-Version: 1.0 -... content-length: 0 -... user-agent: google-cloud-sdk -... accept: application/json -... accept-encoding: gzip, deflate -... Host: compute.googleapis.com -... ''') ->>> msg.attach(base) ->>> msg.as_string() diff --git a/blueprints/cloud-operations/network-dashboard/src/NOTES.md b/blueprints/cloud-operations/network-dashboard/src/NOTES.md deleted file mode 100644 index c73ade11b2..0000000000 --- a/blueprints/cloud-operations/network-dashboard/src/NOTES.md +++ /dev/null @@ -1,70 +0,0 @@ -# Notes - -- [x] get projects - `get_monitored_projects_list` -- [ ] set monitoring interval - `monitoring_interval` -- [ ] read metrics and limits from yaml and create descriptors - `metrics.create_metrics` -- [x] get project quota - `limits.get_quota_project_limit` -- [x] get firewall rules from CAI - `vpc_firewalls.get_firewalls_dict` -- [x] get firewall policies from CAI - `firewall_policies.get_firewall_policies_dict` -- [x] get instances - `instances.get_gce_instance_dict` -- [x] get forwarding rules for L4 ILB - `ilb_fwrules.get_forwarding_rules_dict` -- [x] get forwarding rules for L7 ILB - `ilb_fwrules.get_forwarding_rules_dict` -- [x] get subnets and secondary ranges - `networks.get_subnet_ranges_dict` -- [x] get static routes - `routes.get_static_routes_dict` -- [x] get dynamic routes - routes.get_dynamic_routes - - get routers - `routers.get_routers` - - get networks - `networks.get_networks` - - get router status - `get_routes_for_network` - `get_routes_for_router` -- [x] get and store subnet metrics - `subnets.get_subnets` - - get subnets - `get_all_subnets` - - calculate subnet utilization - `compute_subnet_utilization` - - [x] get instances - `compute_subnet_utilization_vms` - - [x] get forwarding rules - `compute_subnet_utilization_ilbs` - - [x] get addresses - `compute_subnet_utilization_addresses` - - [ ] get redis instances - `compute_subnet_utilization_redis` - - store metrics -- [x]calculate and store firewall rule metrics - `vpc_firewalls.get_firewalls_data` -- [x] calculate and store firewall policy metrics - `firewall_policies.get_firewal_policies_data` -- [x] calculate and store instance per network metrics - `instances.get_gce_instances_data` -- [x] calculate and store L4 forwarding rule metrics - `ilb_fwrules.get_forwarding_rules_data` -- [x] calculate and store L7 forwarding rule metrics - `ilb_fwrules.get_forwarding_rules_data` -- [x] calculate and store static routes metrics - `routes.get_static_routes_data` -- [x] calculate and store peering metrics - `peerings.get_vpc_peering_data` -- [x] calculate and store peering group metrics - `metrics.get_pgg_data` - `routes.get_routes_ppg` -- [ ] write buffered timeseries - `metrics.flush_series_buffer` -- [x] add per-network and per-project hidden quota override - - [x] implement a custom quota override mechanism - - [x] use it in timeseries plugins diff --git a/blueprints/cloud-operations/network-dashboard/src/main.py b/blueprints/cloud-operations/network-dashboard/src/main.py index 39c7f693ba..9a2e2b876e 100755 --- a/blueprints/cloud-operations/network-dashboard/src/main.py +++ b/blueprints/cloud-operations/network-dashboard/src/main.py @@ -14,9 +14,12 @@ # limitations under the License. 'Network dashboard: create network-related metric timeseries for GCP resources.' +import base64 +import binascii import collections import json import logging +import os import click import google.auth @@ -58,6 +61,7 @@ def do_discovery(resources): if result.json: try: # decode the JSON HTTP response and pass it to the plugin + LOGGER.debug(f'passing JSON result to {plugin.name}') results = plugin.func(resources, response, response.json()) except json.decoder.JSONDecodeError as e: LOGGER.critical( @@ -65,10 +69,12 @@ def do_discovery(resources): continue else: # pass the raw HTTP response to the plugin + LOGGER.debug(f'passing raw result to {plugin.name}') results = plugin.func(resources, response) q += collections.deque(results) elif isinstance(result, plugins.Resource): # store a resource the plugin derived from a previous HTTP response + LOGGER.debug(f'got resource {result} from {plugin.name}') if result.key: # this specific resource is indexed by an additional key resources[result.type][result.id][result.key] = result.data @@ -102,6 +108,7 @@ def do_init(resources, discovery_root, op_project, folders=None, projects=None, resources['folders'] = {f: {} for f in folders} for plugin in plugins.get_init_plugins(): plugin.func(resources) + LOGGER.info(f'init completed, resources {resources}') def do_timeseries_calc(resources, descriptors, timeseries, debug_plugin=None): @@ -205,11 +212,46 @@ def fetch(request): return response +def main_cf_pubsub(event, context): + 'Entry point for Cloud Function triggered by a PubSub message.' + debug = os.environ.get('DEBUG') + logging.basicConfig(level=logging.DEBUG if debug else logging.INFO) + LOGGER.info('processing pubsub payload') + try: + payload = json.loads(base64.b64decode(event['data']).decode('utf-8')) + except (binascii.Error, json.JSONDecodeError) as e: + raise SystemExit(f'Invalid payload: e.args[0].') + discovery_root = payload.get('discovery_root') + op_project = payload.get('op_project') + if not discovery_root: + LOGGER.critical('no discovery roo project specified') + LOGGER.info(payload) + raise SystemExit(f'Invalid options') + if not op_project: + LOGGER.critical('no monitoring project specified') + LOGGER.info(payload) + raise SystemExit(f'Invalid options') + if discovery_root.partition('/')[0] not in ('folders', 'organizations'): + raise SystemExit(f'Invalid discovery root {discovery_root}.') + custom_quota = payload.get('custom_quota', {}) + descriptors = [] + folders = payload.get('folders', []) + projects = payload.get('projects', []) + resources = {} + timeseries = [] + do_init(resources, discovery_root, op_project, folders, projects, + custom_quota) + do_discovery(resources) + do_timeseries_calc(resources, descriptors, timeseries) + do_timeseries_descriptors(op_project, resources['metric-descriptors'], + descriptors) + do_timeseries(op_project, timeseries, descriptors) + + @click.command() @click.option( '--discovery-root', '-dr', required=True, - help='Node used for asset discovery, either organizations/nnn or folders/nnn.' -) + help='Root node for asset discovery, organizations/nnn or folders/nnn.') @click.option('--op-project', '-op', required=True, type=str, help='GCP monitoring project where metrics will be stored.') @click.option('--project', '-p', type=str, multiple=True, @@ -224,7 +266,7 @@ def fetch(request): help='Load JSON resources from file, skips init and discovery.') @click.option('--debug-plugin', help='Run only core and specified timeseries plugin.') -def main(discovery_root=None, op_project=None, project=None, folder=None, +def main(discovery_root, op_project, project=None, folder=None, custom_quota_file=None, dump_file=None, load_file=None, debug_plugin=None): 'CLI entry point.' @@ -255,4 +297,4 @@ def main(discovery_root=None, op_project=None, project=None, folder=None, if __name__ == '__main__': - main(auto_envvar_prefix='NETMON') + main_cli(auto_envvar_prefix='NETMON') diff --git a/blueprints/cloud-operations/network-dashboard/src/out.json b/blueprints/cloud-operations/network-dashboard/src/out.json deleted file mode 100644 index 134a8def6d..0000000000 --- a/blueprints/cloud-operations/network-dashboard/src/out.json +++ /dev/null @@ -1,2796 +0,0 @@ -{ - "config:discovery_root": "organizations/436789450919", - "config:monitoring_project": "tf-playground-svpc-net", - "config:folders": [ - "821058723541", - "949988871993", - "321477570496" - ], - "config:projects": [ - "tf-playground-simple" - ], - "config:custom_quota": { - "projects": { - "tf-playground-svpc-net": { - "global": { - "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 - } - } - }, - "networks": { - "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "PEERINGS_PER_NETWORK": 40 - } - } - }, - "config:monitoring_root": "netmon/", - "organization": "436789450919", - "folders": { - "821058723541": { - "name": "Networking" - }, - "949988871993": { - "name": "Security" - }, - "321477570496": { - "name": "Sandbox" - }, - "1027067030104": { - "name": "Development" - }, - "948763653807": { - "name": "Production" - } - }, - "projects": { - "ludo-dev-net-spoke-0": { - "number": "759592912116", - "project_id": "ludo-dev-net-spoke-0" - }, - "ludo-prod-net-landing-0": { - "number": "233262889141", - "project_id": "ludo-prod-net-landing-0" - }, - "ludo-prod-net-spoke-0": { - "number": "195159130008", - "project_id": "ludo-prod-net-spoke-0" - }, - "ludo-dev-sec-core-0": { - "number": "971935141727", - "project_id": "ludo-dev-sec-core-0" - }, - "ludo-prod-sec-core-0": { - "number": "990827422843", - "project_id": "ludo-prod-sec-core-0" - }, - "tf-playground-gcs-test-0": { - "number": "833078410166", - "project_id": "tf-playground-gcs-test-0" - }, - "tf-playground-svpc-gce-dr": { - "number": "433746214138", - "project_id": "tf-playground-svpc-gce-dr" - }, - "tf-playground-svpc-net-dr": { - "number": "697669426824", - "project_id": "tf-playground-svpc-net-dr" - }, - "tf-playground-svpc-openshift": { - "number": "515444627958", - "project_id": "tf-playground-svpc-openshift" - }, - "tf-playground-svpc-gce": { - "number": "783093469136", - "project_id": "tf-playground-svpc-gce" - }, - "tf-playground-svpc-net": { - "number": "1079408472053", - "project_id": "tf-playground-svpc-net" - }, - "tf-playground-svpc-gke": { - "number": "1043465648801", - "project_id": "tf-playground-svpc-gke" - }, - "tf-playground-simple": { - "number": "64297462517", - "project_id": "tf-playground-simple" - } - }, - "projects:number": { - "759592912116": { - "number": "759592912116", - "project_id": "ludo-dev-net-spoke-0" - }, - "233262889141": { - "number": "233262889141", - "project_id": "ludo-prod-net-landing-0" - }, - "195159130008": { - "number": "195159130008", - "project_id": "ludo-prod-net-spoke-0" - }, - "971935141727": { - "number": "971935141727", - "project_id": "ludo-dev-sec-core-0" - }, - "990827422843": { - "number": "990827422843", - "project_id": "ludo-prod-sec-core-0" - }, - "833078410166": { - "number": "833078410166", - "project_id": "tf-playground-gcs-test-0" - }, - "433746214138": { - "number": "433746214138", - "project_id": "tf-playground-svpc-gce-dr" - }, - "697669426824": { - "number": "697669426824", - "project_id": "tf-playground-svpc-net-dr" - }, - "515444627958": { - "number": "515444627958", - "project_id": "tf-playground-svpc-openshift" - }, - "783093469136": { - "number": "783093469136", - "project_id": "tf-playground-svpc-gce" - }, - "1079408472053": { - "number": "1079408472053", - "project_id": "tf-playground-svpc-net" - }, - "1043465648801": { - "number": "1043465648801", - "project_id": "tf-playground-svpc-gke" - }, - "64297462517": { - "number": "64297462517", - "project_id": "tf-playground-simple" - } - }, - "addresses": { - "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello": { - "id": "2569728380045293941", - "name": "psc-home-hello", - "self_link": "projects/tf-playground-simple/regions/europe-west8/addresses/psc-home-hello", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "address": "10.24.33.2", - "internal": true, - "purpose": "GCE_ENDPOINT", - "status": "IN_USE", - "network": null, - "subnetwork": "projects/tf-playground-simple/regions/europe-west8/subnetworks/default" - }, - "projects/ludo-prod-net-landing-0/regions/europe-west1/addresses/dns-forwarding-3bfb2285cc42c149": { - "id": "1086509377706319543", - "name": "dns-forwarding-3bfb2285cc42c149", - "self_link": "projects/ludo-prod-net-landing-0/regions/europe-west1/addresses/dns-forwarding-3bfb2285cc42c149", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "address": "10.128.0.2", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-172afd5c9074083e": { - "id": "7127118736212042670", - "name": "dns-forwarding-172afd5c9074083e", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-172afd5c9074083e", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.0.0.57", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-54cb61094a666746": { - "id": "7222063742818919342", - "name": "dns-forwarding-54cb61094a666746", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-54cb61094a666746", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.0.8.212", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-bd1b9d6abb188fe6": { - "id": "1333839437438854061", - "name": "dns-forwarding-bd1b9d6abb188fe6", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-bd1b9d6abb188fe6", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.0.32.4", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-df6066a8d47814f0": { - "id": "4253770329399436206", - "name": "dns-forwarding-df6066a8d47814f0", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/addresses/dns-forwarding-df6066a8d47814f0", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.0.16.6", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" - }, - "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-79dc52791ec64c76": { - "id": "6033180111970444205", - "name": "dns-forwarding-79dc52791ec64c76", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-79dc52791ec64c76", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.8.0.3", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net" - }, - "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-daf734e63a98deae": { - "id": "569155647513502637", - "name": "dns-forwarding-daf734e63a98deae", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/addresses/dns-forwarding-daf734e63a98deae", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.8.32.4", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce" - }, - "projects/tf-playground-svpc-net/regions/europe-west4/addresses/dns-forwarding-967ac4c986c1c5c3": { - "id": "5851821467174642606", - "name": "dns-forwarding-967ac4c986c1c5c3", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west4/addresses/dns-forwarding-967ac4c986c1c5c3", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.16.32.3", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/addresses/bastion-wg": { - "id": "9106472427605818326", - "name": "bastion-wg", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/bastion-wg", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "34.154.198.58", - "internal": false, - "purpose": "", - "status": "IN_USE", - "network": null, - "subnetwork": null - }, - "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b295f9019fda5f74": { - "id": "42185983700394876", - "name": "dns-forwarding-b295f9019fda5f74", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b295f9019fda5f74", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.24.32.2", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b317a059abd2811d": { - "id": "1218744583122739717", - "name": "dns-forwarding-b317a059abd2811d", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-b317a059abd2811d", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.255.2.2", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-e1617ac9a49c0ec7": { - "id": "4510536153047654869", - "name": "dns-forwarding-e1617ac9a49c0ec7", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/dns-forwarding-e1617ac9a49c0ec7", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.24.0.2", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-auto-ip-8742544-2-1664893602541039": { - "id": "4146356766404377677", - "name": "nat-auto-ip-8742544-2-1664893602541039", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-auto-ip-8742544-2-1664893602541039", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "34.154.124.197", - "internal": false, - "purpose": "NAT_AUTO", - "status": "IN_USE", - "network": null, - "subnetwork": null - }, - "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-ingress-ip-1510-1968901406393301314": { - "id": "9125076950352066911", - "name": "nat-ingress-ip-1510-1968901406393301314", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/addresses/nat-ingress-ip-1510-1968901406393301314", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "172.16.255.2", - "internal": true, - "purpose": "PSC_PRODUCER_NAT_IP", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc" - }, - "projects/tf-playground-svpc-net/regions/us-central1/addresses/dns-forwarding-8aa1f3dca28ea24f": { - "id": "2504084894438347729", - "name": "dns-forwarding-8aa1f3dca28ea24f", - "self_link": "projects/tf-playground-svpc-net/regions/us-central1/addresses/dns-forwarding-8aa1f3dca28ea24f", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.0.9.2", - "internal": true, - "purpose": "DNS_RESOLVER", - "status": "RESERVED", - "network": null, - "subnetwork": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke" - } - }, - "firewall_policies": { - "locations/global/firewallPolicies/104375135909": { - "id": "104375135909", - "name": "104375135909", - "self_link": "locations/global/firewallPolicies/104375135909", - "parent": "folders/821058723541", - "num_rules": 8, - "num_tuples": 24 - } - }, - "firewall_rules": { - "projects/tf-playground-simple/global/firewalls/default-allow-icmp": { - "id": "9189037418640505268", - "name": "default-allow-icmp", - "self_link": "projects/tf-playground-simple/global/firewalls/default-allow-icmp", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "network": "projects/tf-playground-simple/global/networks/default" - }, - "projects/tf-playground-simple/global/firewalls/test-allow-icmp": { - "id": "7916104499034728911", - "name": "test-allow-icmp", - "self_link": "projects/tf-playground-simple/global/firewalls/test-allow-icmp", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "network": "projects/tf-playground-simple/global/networks/test" - }, - "projects/ludo-prod-net-landing-0/global/firewalls/allow-onprem-probes-example": { - "id": "7090352733815889570", - "name": "allow-onprem-probes-example", - "self_link": "projects/ludo-prod-net-landing-0/global/firewalls/allow-onprem-probes-example", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-admins": { - "id": "3420407108972372213", - "name": "shared-vpc-ingress-admins", - "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-admins", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-http": { - "id": "2424070100408918260", - "name": "shared-vpc-ingress-tag-http", - "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-http", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-https": { - "id": "7645864452390445302", - "name": "shared-vpc-ingress-tag-https", - "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-https", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-ssh": { - "id": "6993639888575062262", - "name": "shared-vpc-ingress-tag-ssh", - "self_link": "projects/tf-playground-svpc-net-dr/global/firewalls/shared-vpc-ingress-tag-ssh", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-all": { - "id": "644754961722399058", - "name": "gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-all", - "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-all", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-master": { - "id": "4060019673106897235", - "name": "gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-master", - "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-master", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-vms": { - "id": "1839785476750465362", - "name": "gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-vms", - "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/gke-europe-west1-ludo-dev-data--1f67debe-gke-8dc78f88-vms", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-composer-nodes": { - "id": "717568057687567763", - "name": "ingress-allow-composer-nodes", - "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-composer-nodes", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-dataflow-load": { - "id": "1577493211258187155", - "name": "ingress-allow-dataflow-load", - "self_link": "projects/ludo-dev-net-spoke-0/global/firewalls/ingress-allow-dataflow-load", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/tf-playground-svpc-net/global/firewalls/ilb-l7": { - "id": "3828320867690421959", - "name": "ilb-l7", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/ilb-l7", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/ingress-dns": { - "id": "3282453664652777643", - "name": "ingress-dns", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/ingress-dns", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/ingress-rdp": { - "id": "5733710631021458602", - "name": "ingress-rdp", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/ingress-rdp", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/mikro": { - "id": "7384237133743586341", - "name": "mikro", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/mikro", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/proxy": { - "id": "2052097723271132136", - "name": "proxy", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/proxy", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-admins": { - "id": "6595227897868606454", - "name": "shared-vpc-ingress-admins", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-admins", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-http": { - "id": "565554973513112567", - "name": "shared-vpc-ingress-tag-http", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-http", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-https": { - "id": "1628950298606459895", - "name": "shared-vpc-ingress-tag-https", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-https", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-ssh": { - "id": "365815681166629877", - "name": "shared-vpc-ingress-tag-ssh", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/shared-vpc-ingress-tag-ssh", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/firewalls/test-ingress-echo": { - "id": "1793500710165447459", - "name": "test-ingress-echo", - "self_link": "projects/tf-playground-svpc-net/global/firewalls/test-ingress-echo", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - } - }, - "forwarding_rules": { - "projects/tf-playground-simple/regions/europe-west8/forwardingRules/home-hello": { - "id": "1968901406393301314", - "name": "home-hello", - "self_link": "projects/tf-playground-simple/regions/europe-west8/forwardingRules/home-hello", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "address": "10.24.33.2", - "load_balancing_scheme": "", - "network": "projects/tf-playground-simple/global/networks/default", - "psc_accepted": true, - "region": "europe-west8", - "subnetwork": null - }, - "projects/tf-playground-svpc-gce/regions/europe-west8/forwardingRules/test": { - "id": "5258452901764956944", - "name": "test", - "self_link": "projects/tf-playground-svpc-gce/regions/europe-west8/forwardingRules/test", - "project_id": "tf-playground-svpc-gce", - "project_number": "783093469136", - "address": "10.24.32.33", - "load_balancing_scheme": "INTERNAL", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "psc_accepted": false, - "region": "europe-west8", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/bastion-wg": { - "id": "4724174290265929790", - "name": "bastion-wg", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/bastion-wg", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "34.154.198.58", - "load_balancing_scheme": "EXTERNAL", - "network": null, - "psc_accepted": false, - "region": "europe-west8", - "subnetwork": null - }, - "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/ilb-test": { - "id": "697628543187352862", - "name": "ilb-test", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/forwardingRules/ilb-test", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "address": "10.24.32.29", - "load_balancing_scheme": "INTERNAL_MANAGED", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "psc_accepted": false, - "region": "europe-west8", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - } - }, - "instances": { - "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/nginx-ew8-b": { - "id": "1193081157111798056", - "name": "nginx-ew8-b", - "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/nginx-ew8-b", - "project_id": "tf-playground-svpc-gce", - "project_number": "783093469136", - "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", - "networks": [ - { - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - } - ] - }, - "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-0": { - "id": "8688603441516413730", - "name": "test-0", - "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-0", - "project_id": "tf-playground-svpc-gce", - "project_number": "783093469136", - "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", - "networks": [ - { - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - } - ] - }, - "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-1": { - "id": "6078779822666056482", - "name": "test-1", - "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-b/instances/test-1", - "project_id": "tf-playground-svpc-gce", - "project_number": "783093469136", - "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-b", - "networks": [ - { - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - } - ] - }, - "projects/tf-playground-svpc-gce/zones/europe-west8-c/instances/nginx-ew8-c": { - "id": "4082958655368747304", - "name": "nginx-ew8-c", - "self_link": "projects/tf-playground-svpc-gce/zones/europe-west8-c/instances/nginx-ew8-c", - "project_id": "tf-playground-svpc-gce", - "project_number": "783093469136", - "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-gce/zones/europe-west8-c", - "networks": [ - { - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce" - } - ] - }, - "projects/tf-playground-svpc-net/zones/europe-west8-b/instances/bastion": { - "id": "837293213121504385", - "name": "bastion", - "self_link": "projects/tf-playground-svpc-net/zones/europe-west8-b/instances/bastion", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "zone": "https://www.googleapis.com/compute/v1/projects/tf-playground-svpc-net/zones/europe-west8-b", - "networks": [ - { - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "subnetwork": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net" - } - ] - } - }, - "networks": { - "projects/tf-playground-simple/global/networks/default": { - "id": "8227259653970917801", - "name": "default", - "self_link": "projects/tf-playground-simple/global/networks/default", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "peerings": [], - "subnetworks": [ - "projects/tf-playground-simple/regions/europe-west8/subnetworks/default" - ] - }, - "projects/tf-playground-simple/global/networks/test": { - "id": "3222287547582786016", - "name": "test", - "self_link": "projects/tf-playground-simple/global/networks/test", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "peerings": [], - "subnetworks": [ - "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default" - ] - }, - "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0": { - "id": "2315184905594658556", - "name": "prod-spoke-0", - "self_link": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "peerings": [ - { - "active": true, - "name": "prod-peering-0-prod-spoke-0-prod-landing-0", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", - "project_id": "ludo-prod-net-landing-0" - }, - { - "active": true, - "name": "servicenetworking-googleapis-com", - "network": "projects/r63cc730008bdfb49p-tp/global/networks/servicenetworking", - "project_id": "r63cc730008bdfb49p-tp" - }, - { - "active": true, - "name": "to-dev-spoke-0", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "project_id": "ludo-dev-net-spoke-0" - }, - { - "active": false, - "name": "to-test-public-gcs", - "network": "projects/186042149622/global/networks/default", - "project_id": "186042149622" - } - ], - "subnetworks": [ - "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4", - "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1", - "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4", - "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1" - ] - }, - "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0": { - "id": "2001134848211738357", - "name": "prod-landing-0", - "self_link": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "peerings": [ - { - "active": true, - "name": "dev-peering-0-prod-landing-0-dev-spoke-0", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "project_id": "ludo-dev-net-spoke-0" - }, - { - "active": true, - "name": "prod-peering-0-prod-landing-0-prod-spoke-0", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "project_id": "ludo-prod-net-spoke-0" - } - ], - "subnetworks": [ - "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1" - ] - }, - "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc": { - "id": "1887910606791828228", - "name": "shared-vpc", - "self_link": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "peerings": [], - "subnetworks": [ - "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb", - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net", - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip", - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce", - "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke", - "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce", - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke", - "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce", - "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce", - "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net", - "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net" - ] - }, - "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0": { - "id": "5618184304347635448", - "name": "dev-spoke-0", - "self_link": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "peerings": [ - { - "active": false, - "name": "composer-474420912893-vpc-peering", - "network": "projects/474420912893/global/networks/composer-net", - "project_id": "474420912893" - }, - { - "active": false, - "name": "composer-566526877645-vpc-peering", - "network": "projects/566526877645/global/networks/composer-net", - "project_id": "566526877645" - }, - { - "active": true, - "name": "dev-peering-0-dev-spoke-0-prod-landing-0", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", - "project_id": "ludo-prod-net-landing-0" - }, - { - "active": false, - "name": "gke-n0a46a71e52c28142071-9ca9-f42d-peer", - "network": "projects/gke-prod-europe-west1-b-4/global/networks/gke-n0a46a71e52c28142071-9ca9-32cb-net", - "project_id": "gke-prod-europe-west1-b-4" - }, - { - "active": true, - "name": "servicenetworking-googleapis-com", - "network": "projects/p10d8c97c9bd39799p-tp/global/networks/servicenetworking", - "project_id": "p10d8c97c9bd39799p-tp" - }, - { - "active": true, - "name": "to-prod-spoke-0", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "project_id": "ludo-prod-net-spoke-0" - } - ], - "subnetworks": [ - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1", - "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8", - "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4", - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1", - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1", - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1" - ] - }, - "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "id": "6490252338739305443", - "name": "shared-vpc", - "self_link": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "peerings": [ - { - "active": true, - "name": "servicenetworking-googleapis-com", - "network": "projects/a2fed18bfde5785bdp-tp/global/networks/servicenetworking", - "project_id": "a2fed18bfde5785bdp-tp" - } - ], - "subnetworks": [ - "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce", - "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke", - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net", - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce", - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net", - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc", - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke", - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb", - "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net", - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce", - "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce", - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip" - ] - } - }, - "subnetworks": { - "projects/tf-playground-simple/regions/europe-west8/subnetworks/default": { - "id": "521276777991436721", - "name": "default", - "self_link": "projects/tf-playground-simple/regions/europe-west8/subnetworks/default", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "cidr_range": "10.24.33.0/24", - "network": "projects/tf-playground-simple/global/networks/default", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default": { - "id": "3192067775436040655", - "name": "test-default", - "self_link": "projects/tf-playground-simple/regions/europe-west8/subnetworks/test-default", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "cidr_range": "10.0.0.0/24", - "network": "projects/tf-playground-simple/global/networks/test", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1": { - "id": "4645967677499694788", - "name": "prod-default-ew1", - "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-default-ew1", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "cidr_range": "10.128.64.0/24", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1": { - "id": "5412553412046524467", - "name": "prod-l7ilb-europe-west1", - "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west1/subnetworks/prod-l7ilb-europe-west1", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "cidr_range": "10.128.92.0/24", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "purpose": "REGIONAL_MANAGED_PROXY", - "region": "europe-west1" - }, - "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4": { - "id": "6980100425062399320", - "name": "prod-default-ew4", - "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-default-ew4", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "cidr_range": "10.128.65.0/24", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "purpose": "PRIVATE", - "region": "europe-west4" - }, - "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4": { - "id": "6293569359006437427", - "name": "prod-l7ilb-europe-west4", - "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west4/subnetworks/prod-l7ilb-europe-west4", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "cidr_range": "10.128.93.0/24", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "purpose": "REGIONAL_MANAGED_PROXY", - "region": "europe-west4" - }, - "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1": { - "id": "6183073250819497646", - "name": "landing-default-ew1", - "self_link": "projects/ludo-prod-net-landing-0/regions/europe-west1/subnetworks/landing-default-ew1", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "cidr_range": "10.128.0.0/24", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce": { - "id": "1753894793794347243", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gce", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.0.32.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke": { - "id": "4511601755917967596", - "name": "gke", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.0.8.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip": { - "id": "5701824999950967019", - "name": "gke-vip", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/gke-vip", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.0.16.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net": { - "id": "2784995877536981227", - "name": "net", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west1/subnetworks/net", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.0.0.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce": { - "id": "4654054937679159533", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/gce", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.8.32.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west3" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net": { - "id": "1999120736153034998", - "name": "net", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west3/subnetworks/net", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.8.0.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west3" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce": { - "id": "4593147538303148267", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west4/subnetworks/gce", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.16.32.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west4" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce": { - "id": "7339386710113181931", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/gce", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.24.32.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb": { - "id": "137392004069122283", - "name": "l7ilb", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/l7ilb", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.255.2.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "REGIONAL_MANAGED_PROXY", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net": { - "id": "3309179009467998443", - "name": "net", - "self_link": "projects/tf-playground-svpc-net-dr/regions/europe-west8/subnetworks/net", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.24.0.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke": { - "id": "748427805746061548", - "name": "gke", - "self_link": "projects/tf-playground-svpc-net-dr/regions/us-central1/subnetworks/gke", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "cidr_range": "10.0.9.0/24", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "us-central1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1": { - "id": "6460040680782777531", - "name": "dev-dataplatform-ew1", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-dataplatform-ew1", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "cidr_range": "10.128.48.0/24", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1": { - "id": "2831390316161982168", - "name": "dev-default-ew1", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-default-ew1", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "cidr_range": "10.128.32.0/24", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1": { - "id": "249944920601617510", - "name": "dev-gke-nodes-ew1", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "cidr_range": "10.64.0.0/24", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1": { - "id": "2280726911218651187", - "name": "dev-l7ilb-europe-west1", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-l7ilb-europe-west1", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "cidr_range": "10.128.60.0/24", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "purpose": "REGIONAL_MANAGED_PROXY", - "region": "europe-west1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4": { - "id": "6599648524023664689", - "name": "dev-l7ilb-europe-west4", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west4/subnetworks/dev-l7ilb-europe-west4", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "cidr_range": "10.128.61.0/24", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "purpose": "REGIONAL_MANAGED_PROXY", - "region": "europe-west4" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8": { - "id": "4456740759944235703", - "name": "ludo-dev-default-ew8", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west8/subnetworks/ludo-dev-default-ew8", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "cidr_range": "10.128.33.0/24", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce": { - "id": "1129946310421719129", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gce", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.0.32.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke": { - "id": "2478860073774111734", - "name": "gke", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.0.8.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip": { - "id": "1416395609824617176", - "name": "gke-vip", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/gke-vip", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.0.16.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net": { - "id": "8424841432903474166", - "name": "net", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/subnetworks/net", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.0.0.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce": { - "id": "8930552426943163299", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/gce", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.8.32.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west3" - }, - "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net": { - "id": "5182660640425322036", - "name": "net", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/subnetworks/net", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.8.0.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west3" - }, - "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce": { - "id": "2464381379294734425", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west4/subnetworks/gce", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.16.32.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west4" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce": { - "id": "6421800903324885909", - "name": "gce", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/gce", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.24.32.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb": { - "id": "9159849323393344035", - "name": "l7ilb", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/l7ilb", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.255.2.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "REGIONAL_MANAGED_PROXY", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net": { - "id": "4872843329097064947", - "name": "net", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/net", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.24.0.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc": { - "id": "1816076316698934074", - "name": "psc", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/subnetworks/psc", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "172.16.255.0/25", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE_SERVICE_CONNECT", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke": { - "id": "440151099218547687", - "name": "gke", - "self_link": "projects/tf-playground-svpc-net/regions/us-central1/subnetworks/gke", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "cidr_range": "10.0.9.0/24", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "purpose": "PRIVATE", - "region": "us-central1" - } - }, - "routers": { - "projects/ludo-prod-net-spoke-0/regions/europe-west1/routers/prod-nat-ew1-nat": { - "id": "983301949238599311", - "name": "prod-nat-ew1-nat", - "self_link": "projects/ludo-prod-net-spoke-0/regions/europe-west1/routers/prod-nat-ew1-nat", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0", - "region": "europe-west1" - }, - "projects/ludo-prod-net-landing-0/regions/europe-west1/routers/prod-nat-ew1": { - "id": "478658513322696364", - "name": "prod-nat-ew1", - "self_link": "projects/ludo-prod-net-landing-0/regions/europe-west1/routers/prod-nat-ew1", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0", - "region": "europe-west1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west1/routers/dev-nat-ew1-nat": { - "id": "2223705014177685131", - "name": "dev-nat-ew1-nat", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west1/routers/dev-nat-ew1-nat", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "region": "europe-west1" - }, - "projects/ludo-dev-net-spoke-0/regions/europe-west8/routers/dev-nat-ew8-nat": { - "id": "5947823903261526678", - "name": "dev-nat-ew8-nat", - "self_link": "projects/ludo-dev-net-spoke-0/regions/europe-west8/routers/dev-nat-ew8-nat", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/europe-west1/routers/vpc-shared-ew1-nat": { - "id": "3113194381827127169", - "name": "vpc-shared-ew1-nat", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west1/routers/vpc-shared-ew1-nat", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "region": "europe-west1" - }, - "projects/tf-playground-svpc-net/regions/europe-west3/routers/vpc-shared-ew3-nat": { - "id": "8873531324389668767", - "name": "vpc-shared-ew3-nat", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west3/routers/vpc-shared-ew3-nat", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "region": "europe-west3" - }, - "projects/tf-playground-svpc-net/regions/europe-west4/routers/vpc-shared-ew4-nat": { - "id": "6860218593541109633", - "name": "vpc-shared-ew4-nat", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west4/routers/vpc-shared-ew4-nat", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "region": "europe-west4" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpc-shared-ew8-nat": { - "id": "7968307069143758750", - "name": "vpc-shared-ew8-nat", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpc-shared-ew8-nat", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpn-home": { - "id": "3925724850449128568", - "name": "vpn-home", - "self_link": "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpn-home", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "region": "europe-west8" - }, - "projects/tf-playground-svpc-net/regions/us-central1/routers/vpc-shared-uc1-nat": { - "id": "441772775809234847", - "name": "vpc-shared-uc1-nat", - "self_link": "projects/tf-playground-svpc-net/regions/us-central1/routers/vpc-shared-uc1-nat", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc", - "region": "us-central1" - } - }, - "routes": { - "projects/tf-playground-simple/global/routes/default-route-1e14cf2fe4b89f0b": { - "id": "1065219576982546940", - "name": "default-route-1e14cf2fe4b89f0b", - "self_link": "projects/tf-playground-simple/global/routes/default-route-1e14cf2fe4b89f0b", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "next_hop_type": "gateway", - "network": "projects/tf-playground-simple/global/networks/test" - }, - "projects/tf-playground-simple/global/routes/default-route-709ad165b8383d89": { - "id": "1159999789989296547", - "name": "default-route-709ad165b8383d89", - "self_link": "projects/tf-playground-simple/global/routes/default-route-709ad165b8383d89", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "next_hop_type": "gateway", - "network": "projects/tf-playground-simple/global/networks/default" - }, - "projects/tf-playground-simple/global/routes/default-route-8706754714306526": { - "id": "1359944480556775819", - "name": "default-route-8706754714306526", - "self_link": "projects/tf-playground-simple/global/routes/default-route-8706754714306526", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "next_hop_type": "network", - "network": "projects/tf-playground-simple/global/networks/default" - }, - "projects/tf-playground-simple/global/routes/default-route-94ba9f38b7d02917": { - "id": "6942400975983749580", - "name": "default-route-94ba9f38b7d02917", - "self_link": "projects/tf-playground-simple/global/routes/default-route-94ba9f38b7d02917", - "project_id": "tf-playground-simple", - "project_number": "64297462517", - "next_hop_type": "network", - "network": "projects/tf-playground-simple/global/networks/test" - }, - "projects/ludo-prod-net-spoke-0/global/routes/default-route-40f9b3cd9946750d": { - "id": "899011966177939471", - "name": "default-route-40f9b3cd9946750d", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-40f9b3cd9946750d", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "network", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/default-route-72ef5fa46984f071": { - "id": "8568498928831487711", - "name": "default-route-72ef5fa46984f071", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-72ef5fa46984f071", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "network", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/default-route-ac3bcc52856e80e3": { - "id": "1784782880021188656", - "name": "default-route-ac3bcc52856e80e3", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-ac3bcc52856e80e3", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "network", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/default-route-c51a2cb0324f01a9": { - "id": "2824182523471707477", - "name": "default-route-c51a2cb0324f01a9", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-c51a2cb0324f01a9", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "network", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/default-route-f1fd5dec0cdd76f3": { - "id": "308922196058758903", - "name": "default-route-f1fd5dec0cdd76f3", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/default-route-f1fd5dec0cdd76f3", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "gateway", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-0321109d8d96dc9e": { - "id": "3324799109720031011", - "name": "peering-route-0321109d8d96dc9e", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-0321109d8d96dc9e", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-1644420b9cb80ef8": { - "id": "6241664760832264842", - "name": "peering-route-1644420b9cb80ef8", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-1644420b9cb80ef8", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-25f1911afe289c3c": { - "id": "6827131325114597155", - "name": "peering-route-25f1911afe289c3c", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-25f1911afe289c3c", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-578ffc0543e989eb": { - "id": "44456750104929401", - "name": "peering-route-578ffc0543e989eb", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-578ffc0543e989eb", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-780e3930f3b7f24c": { - "id": "5746981242429989896", - "name": "peering-route-780e3930f3b7f24c", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-780e3930f3b7f24c", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e997f107eeecfb2f": { - "id": "8585255017838395171", - "name": "peering-route-e997f107eeecfb2f", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e997f107eeecfb2f", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e9d68888bc28ea31": { - "id": "1139893304295800611", - "name": "peering-route-e9d68888bc28ea31", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/peering-route-e9d68888bc28ea31", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-private-googleapis": { - "id": "2948233411096010438", - "name": "prod-spoke-0-private-googleapis", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-private-googleapis", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "gateway", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-restricted-googleapis": { - "id": "6714004201574293210", - "name": "prod-spoke-0-restricted-googleapis", - "self_link": "projects/ludo-prod-net-spoke-0/global/routes/prod-spoke-0-restricted-googleapis", - "project_id": "ludo-prod-net-spoke-0", - "project_number": "195159130008", - "next_hop_type": "gateway", - "network": "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/default-route-ad0e530b51126577": { - "id": "6241659907566788261", - "name": "default-route-ad0e530b51126577", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/default-route-ad0e530b51126577", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "network", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/default-route-b9c9da9f17272c9c": { - "id": "7146043362834964209", - "name": "default-route-b9c9da9f17272c9c", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/default-route-b9c9da9f17272c9c", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "gateway", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-13de4d911be51ea7": { - "id": "3094272727316390945", - "name": "peering-route-13de4d911be51ea7", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-13de4d911be51ea7", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-5401900b4cf2582d": { - "id": "407360634550707234", - "name": "peering-route-5401900b4cf2582d", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-5401900b4cf2582d", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-5914b2470e809033": { - "id": "89240550193980449", - "name": "peering-route-5914b2470e809033", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-5914b2470e809033", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-60914e501bc4499b": { - "id": "5203468670312866824", - "name": "peering-route-60914e501bc4499b", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-60914e501bc4499b", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-78f9f9fb7e76b2c3": { - "id": "4743569086995194914", - "name": "peering-route-78f9f9fb7e76b2c3", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-78f9f9fb7e76b2c3", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-7b4a6991ec5526c2": { - "id": "9040733271652000776", - "name": "peering-route-7b4a6991ec5526c2", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-7b4a6991ec5526c2", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-8c33217d76de1f1b": { - "id": "8099720905059427336", - "name": "peering-route-8c33217d76de1f1b", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-8c33217d76de1f1b", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-93ff9a0739edb2cb": { - "id": "5236433849979724808", - "name": "peering-route-93ff9a0739edb2cb", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-93ff9a0739edb2cb", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-ed15dbfa7467b3cf": { - "id": "3577867237184201761", - "name": "peering-route-ed15dbfa7467b3cf", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-ed15dbfa7467b3cf", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/peering-route-ef500c56c5474741": { - "id": "1283494745588327458", - "name": "peering-route-ef500c56c5474741", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/peering-route-ef500c56c5474741", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "peering", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-private-googleapis": { - "id": "1170308089481584291", - "name": "prod-landing-0-private-googleapis", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-private-googleapis", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "gateway", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-restricted-googleapis": { - "id": "6281371948279717551", - "name": "prod-landing-0-restricted-googleapis", - "self_link": "projects/ludo-prod-net-landing-0/global/routes/prod-landing-0-restricted-googleapis", - "project_id": "ludo-prod-net-landing-0", - "project_number": "233262889141", - "next_hop_type": "gateway", - "network": "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-11fa55a9adcb4b70": { - "id": "4585803715457638631", - "name": "default-route-11fa55a9adcb4b70", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-11fa55a9adcb4b70", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-14ce7cda556cb5eb": { - "id": "5125715850850263272", - "name": "default-route-14ce7cda556cb5eb", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-14ce7cda556cb5eb", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-2c2bb1c74a41b25d": { - "id": "3285216510750286050", - "name": "default-route-2c2bb1c74a41b25d", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-2c2bb1c74a41b25d", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-4bbf918c1bea1feb": { - "id": "2274013185856299239", - "name": "default-route-4bbf918c1bea1feb", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-4bbf918c1bea1feb", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-72dd143284332b85": { - "id": "4671000443382718694", - "name": "default-route-72dd143284332b85", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-72dd143284332b85", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-769a73de008d7193": { - "id": "7621429696248733926", - "name": "default-route-769a73de008d7193", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-769a73de008d7193", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-8c42e476440a1bcb": { - "id": "7399270554358046953", - "name": "default-route-8c42e476440a1bcb", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-8c42e476440a1bcb", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-936ac84acce6cd86": { - "id": "2775823425120559336", - "name": "default-route-936ac84acce6cd86", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-936ac84acce6cd86", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-a14eb190bc5b6e46": { - "id": "2616984457793822950", - "name": "default-route-a14eb190bc5b6e46", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-a14eb190bc5b6e46", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-a805f525462ef124": { - "id": "3493013866118701287", - "name": "default-route-a805f525462ef124", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-a805f525462ef124", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-afaa2cc263a7bfec": { - "id": "5888856166968386334", - "name": "default-route-afaa2cc263a7bfec", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-afaa2cc263a7bfec", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "gateway", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-b3acb1437d77cd80": { - "id": "5120529042185183462", - "name": "default-route-b3acb1437d77cd80", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-b3acb1437d77cd80", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-d8ec4a849124c7a0": { - "id": "1115146042057482473", - "name": "default-route-d8ec4a849124c7a0", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-d8ec4a849124c7a0", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-d9d6128120ab85fc": { - "id": "6526128012596142311", - "name": "default-route-d9d6128120ab85fc", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-d9d6128120ab85fc", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-deb1eff327532f1f": { - "id": "7152080901412020456", - "name": "default-route-deb1eff327532f1f", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-deb1eff327532f1f", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-eaa61c921aba25b0": { - "id": "3430978466596839679", - "name": "default-route-eaa61c921aba25b0", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-eaa61c921aba25b0", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/default-route-eccab6ddfa3b0515": { - "id": "93366519110885618", - "name": "default-route-eccab6ddfa3b0515", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/default-route-eccab6ddfa3b0515", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net-dr/global/routes/shared-vpc-private-googleapis": { - "id": "4227585992519114998", - "name": "shared-vpc-private-googleapis", - "self_link": "projects/tf-playground-svpc-net-dr/global/routes/shared-vpc-private-googleapis", - "project_id": "tf-playground-svpc-net-dr", - "project_number": "697669426824", - "next_hop_type": "gateway", - "network": "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-1e66f6999d63bbdb": { - "id": "2794396208018312351", - "name": "default-route-1e66f6999d63bbdb", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-1e66f6999d63bbdb", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "network", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-25f5b67fa9b5322e": { - "id": "3153137358666260143", - "name": "default-route-25f5b67fa9b5322e", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-25f5b67fa9b5322e", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "network", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-4a1b1a9528dc9066": { - "id": "662362691274237026", - "name": "default-route-4a1b1a9528dc9066", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-4a1b1a9528dc9066", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "network", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-62198dccd5e11e45": { - "id": "2384952332828478130", - "name": "default-route-62198dccd5e11e45", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-62198dccd5e11e45", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "network", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-bbd81d101d7db2bc": { - "id": "4717561835935272974", - "name": "default-route-bbd81d101d7db2bc", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-bbd81d101d7db2bc", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "network", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-c4da502c04154f5a": { - "id": "7479509588678598387", - "name": "default-route-c4da502c04154f5a", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-c4da502c04154f5a", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "gateway", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/default-route-f42e36a70e595e65": { - "id": "1909548583298568205", - "name": "default-route-f42e36a70e595e65", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/default-route-f42e36a70e595e65", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "network", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-private-googleapis": { - "id": "1142085726235582125", - "name": "dev-spoke-0-private-googleapis", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-private-googleapis", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "gateway", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-restricted-googleapis": { - "id": "994791119899385561", - "name": "dev-spoke-0-restricted-googleapis", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/dev-spoke-0-restricted-googleapis", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "gateway", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/peering-route-3d010c7ff6100be5": { - "id": "3538065964205980974", - "name": "peering-route-3d010c7ff6100be5", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-3d010c7ff6100be5", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "peering", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/peering-route-4272e533f3961e2c": { - "id": "2081363471452454691", - "name": "peering-route-4272e533f3961e2c", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-4272e533f3961e2c", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "peering", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/peering-route-47726de7b73f0892": { - "id": "2243680524795077666", - "name": "peering-route-47726de7b73f0892", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-47726de7b73f0892", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "peering", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/peering-route-90e8310f8ac976df": { - "id": "1926683561476603683", - "name": "peering-route-90e8310f8ac976df", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-90e8310f8ac976df", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "peering", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/ludo-dev-net-spoke-0/global/routes/peering-route-9da568659960707b": { - "id": "3178237314998845219", - "name": "peering-route-9da568659960707b", - "self_link": "projects/ludo-dev-net-spoke-0/global/routes/peering-route-9da568659960707b", - "project_id": "ludo-dev-net-spoke-0", - "project_number": "759592912116", - "next_hop_type": "peering", - "network": "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - }, - "projects/tf-playground-svpc-net/global/routes/default": { - "id": "7660236194664304033", - "name": "default", - "self_link": "projects/tf-playground-svpc-net/global/routes/default", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "gateway", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-00276cedf07156e1": { - "id": "5814005392503551500", - "name": "default-route-00276cedf07156e1", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-00276cedf07156e1", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-1640724c624a81a9": { - "id": "7924010442940425742", - "name": "default-route-1640724c624a81a9", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-1640724c624a81a9", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-1a63661f5467170b": { - "id": "7771813397673697776", - "name": "default-route-1a63661f5467170b", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-1a63661f5467170b", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-3968f730f137e758": { - "id": "5204039853703324562", - "name": "default-route-3968f730f137e758", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-3968f730f137e758", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-406dce5c3cd518ae": { - "id": "477834944705155438", - "name": "default-route-406dce5c3cd518ae", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-406dce5c3cd518ae", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-45df00aec802e6e1": { - "id": "858206022063643636", - "name": "default-route-45df00aec802e6e1", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-45df00aec802e6e1", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-46edb716dee26f2d": { - "id": "1194245977988092270", - "name": "default-route-46edb716dee26f2d", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-46edb716dee26f2d", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-62849020e7572a5a": { - "id": "41053546903447638", - "name": "default-route-62849020e7572a5a", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-62849020e7572a5a", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-648effc960cb3912": { - "id": "3109902933644737334", - "name": "default-route-648effc960cb3912", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-648effc960cb3912", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-6d41e89bfb9eb2f6": { - "id": "1628999936569849356", - "name": "default-route-6d41e89bfb9eb2f6", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-6d41e89bfb9eb2f6", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-6e55995b3ecdce42": { - "id": "2529095030201726477", - "name": "default-route-6e55995b3ecdce42", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-6e55995b3ecdce42", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-7055cdd9fb57cccd": { - "id": "345479256826663886", - "name": "default-route-7055cdd9fb57cccd", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-7055cdd9fb57cccd", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-939636f9c9827425": { - "id": "6330256629342462679", - "name": "default-route-939636f9c9827425", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-939636f9c9827425", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-adfa72880afd062f": { - "id": "3946321220088521248", - "name": "default-route-adfa72880afd062f", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-adfa72880afd062f", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-bd832fca2633024d": { - "id": "3512752011138509755", - "name": "default-route-bd832fca2633024d", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-bd832fca2633024d", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-e618f80d606cfa58": { - "id": "4078275391946775500", - "name": "default-route-e618f80d606cfa58", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-e618f80d606cfa58", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/default-route-fa56ade9f222917e": { - "id": "2528926229423401046", - "name": "default-route-fa56ade9f222917e", - "self_link": "projects/tf-playground-svpc-net/global/routes/default-route-fa56ade9f222917e", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "network", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/iap-ig": { - "id": "3009389033799183115", - "name": "iap-ig", - "self_link": "projects/tf-playground-svpc-net/global/routes/iap-ig", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "gateway", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/peering-route-5686ff94bce71953": { - "id": "4281803293141324207", - "name": "peering-route-5686ff94bce71953", - "self_link": "projects/tf-playground-svpc-net/global/routes/peering-route-5686ff94bce71953", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "peering", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - }, - "projects/tf-playground-svpc-net/global/routes/shared-vpc-private-googleapis": { - "id": "8851580889041732346", - "name": "shared-vpc-private-googleapis", - "self_link": "projects/tf-playground-svpc-net/global/routes/shared-vpc-private-googleapis", - "project_id": "tf-playground-svpc-net", - "project_number": "1079408472053", - "next_hop_type": "gateway", - "network": "projects/tf-playground-svpc-net/global/networks/shared-vpc" - } - }, - "quota": { - "ludo-dev-net-spoke-0": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "ludo-prod-net-landing-0": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "ludo-prod-net-spoke-0": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "ludo-dev-sec-core-0": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "ludo-prod-sec-core-0": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-gcs-test-0": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-svpc-gce-dr": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-svpc-net-dr": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-svpc-openshift": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-svpc-gce": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 9, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-svpc-net": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000, - "INTERNAL_FORWARDING_RULES_PER_NETWORK": 750 - } - }, - "tf-playground-svpc-gke": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "tf-playground-simple": { - "global": { - "BACKEND_BUCKETS": 9, - "BACKEND_SERVICES": 75, - "EXTERNAL_VPN_GATEWAYS": 15, - "FIREWALLS": 200, - "FORWARDING_RULES": 45, - "GLOBAL_EXTERNAL_MANAGED_FORWARDING_RULES": 45, - "GLOBAL_INTERNAL_ADDRESSES": 5000, - "HEALTH_CHECKS": 150, - "IMAGES": 2000, - "INSTANCE_TEMPLATES": 300, - "INTERCONNECTS": 6, - "INTERNAL_TRAFFIC_DIRECTOR_FORWARDING_RULES": 45, - "IN_USE_ADDRESSES": 69, - "MACHINE_IMAGES": 2000, - "NETWORKS": 15, - "NETWORK_ENDPOINT_GROUPS": 300, - "NETWORK_FIREWALL_POLICIES": 30, - "PACKET_MIRRORINGS": 45, - "PUBLIC_ADVERTISED_PREFIXES": 1, - "PUBLIC_DELEGATED_PREFIXES": 10, - "ROUTERS": 10, - "ROUTES": 250, - "SECURITY_POLICIES": 10, - "SECURITY_POLICY_CEVAL_RULES": 20, - "SECURITY_POLICY_RULES": 100, - "SNAPSHOTS": 5000, - "SSL_CERTIFICATES": 30, - "STATIC_ADDRESSES": 21, - "STATIC_BYOIP_ADDRESSES": 1024, - "SUBNETWORKS": 175, - "TARGET_HTTPS_PROXIES": 30, - "TARGET_HTTP_PROXIES": 30, - "TARGET_INSTANCES": 150, - "TARGET_POOLS": 150, - "TARGET_SSL_PROXIES": 30, - "TARGET_TCP_PROXIES": 30, - "TARGET_VPN_GATEWAYS": 15, - "URL_MAPS": 30, - "VPN_GATEWAYS": 15, - "VPN_TUNNELS": 30, - "XPN_SERVICE_PROJECTS": 1000 - } - }, - "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "PEERINGS_PER_NETWORK": 40 - } - }, - "routes_dynamic": { - "projects/tf-playground-svpc-net/global/networks/shared-vpc": { - "projects/tf-playground-svpc-net/regions/europe-west8/routers/vpn-home": 1 - } - }, - "networks:project": { - "tf-playground-simple": [ - "projects/tf-playground-simple/global/networks/default", - "projects/tf-playground-simple/global/networks/test" - ], - "ludo-prod-net-spoke-0": [ - "projects/ludo-prod-net-spoke-0/global/networks/prod-spoke-0" - ], - "ludo-prod-net-landing-0": [ - "projects/ludo-prod-net-landing-0/global/networks/prod-landing-0" - ], - "tf-playground-svpc-net-dr": [ - "projects/tf-playground-svpc-net-dr/global/networks/shared-vpc" - ], - "ludo-dev-net-spoke-0": [ - "projects/ludo-dev-net-spoke-0/global/networks/dev-spoke-0" - ], - "tf-playground-svpc-net": [ - "projects/tf-playground-svpc-net/global/networks/shared-vpc" - ] - }, - "metric-descriptors": { - "custom.googleapis.com/netmon/firewall_policy/tuples_available": {}, - "custom.googleapis.com/netmon/firewall_policy/tuples_used": {}, - "custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio": {}, - "custom.googleapis.com/netmon/network/firewall_rules_used": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l4_available": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l4_used": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l7_available": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l7_used": {}, - "custom.googleapis.com/netmon/network/forwarding_rules_l7_used_ratio": {}, - "custom.googleapis.com/netmon/network/instances_available": {}, - "custom.googleapis.com/netmon/network/instances_used": {}, - "custom.googleapis.com/netmon/network/instances_used_ratio": {}, - "custom.googleapis.com/netmon/network/peerings_active_available": {}, - "custom.googleapis.com/netmon/network/peerings_active_used": {}, - "custom.googleapis.com/netmon/network/peerings_active_used_ratio": {}, - "custom.googleapis.com/netmon/network/peerings_total_available": {}, - "custom.googleapis.com/netmon/network/peerings_total_used": {}, - "custom.googleapis.com/netmon/network/peerings_total_used_ratio": {}, - "custom.googleapis.com/netmon/network/routes_dynamic_available": {}, - "custom.googleapis.com/netmon/network/routes_dynamic_used": {}, - "custom.googleapis.com/netmon/network/routes_dynamic_used_ratio": {}, - "custom.googleapis.com/netmon/network/routes_static_used": {}, - "custom.googleapis.com/netmon/network/subnets_available": {}, - "custom.googleapis.com/netmon/network/subnets_used": {}, - "custom.googleapis.com/netmon/network/subnets_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_available": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_available": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used": {}, - "custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/instances_available": {}, - "custom.googleapis.com/netmon/peering_group/instances_used": {}, - "custom.googleapis.com/netmon/peering_group/instances_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/routes_dynamic_available": {}, - "custom.googleapis.com/netmon/peering_group/routes_dynamic_used": {}, - "custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio": {}, - "custom.googleapis.com/netmon/peering_group/routes_static_available": {}, - "custom.googleapis.com/netmon/peering_group/routes_static_used": {}, - "custom.googleapis.com/netmon/peering_group/routes_static_used_ratio": {}, - "custom.googleapis.com/netmon/project/firewall_rules_available": {}, - "custom.googleapis.com/netmon/project/firewall_rules_used": {}, - "custom.googleapis.com/netmon/project/firewall_rules_used_ratio": {}, - "custom.googleapis.com/netmon/project/routes_dynamic_available": {}, - "custom.googleapis.com/netmon/project/routes_dynamic_used": {}, - "custom.googleapis.com/netmon/project/routes_dynamic_used_ratio": {}, - "custom.googleapis.com/netmon/project/routes_static_available": {}, - "custom.googleapis.com/netmon/project/routes_static_used": {}, - "custom.googleapis.com/netmon/project/routes_static_used_ratio": {}, - "custom.googleapis.com/netmon/subnetwork/addresses_available": {}, - "custom.googleapis.com/netmon/subnetwork/addresses_used": {}, - "custom.googleapis.com/netmon/subnetwork/addresses_used_ratio": {} - } -} \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py index 94a5ed18fb..dc5c53247e 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/core-discover-cai-nodes.py @@ -46,6 +46,8 @@ def _handle_discovery(resources, response, data): data = {'number': number, 'project_id': name} yield Resource('projects', name, data) yield Resource('projects:number', number, data) + else: + LOGGER.info(f'unknown resource {name}') next_url = parse_page_token(data, response.request.url) if next_url: LOGGER.info('discovery next url') diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py index 77269c000b..a22b5d644b 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py @@ -44,6 +44,20 @@ NAMES = {v: k for k, v in TYPES.items()} +def _get_parent(parent, resources): + 'Extracts and returns resource parent and type.' + parent_type, parent_id = parent.split('/')[-2:] + if parent_type == 'projects': + project = resources['projects:number'].get(parent_id) + if project: + return {'project_id': project['project_id'], 'project_number': parent_id} + if parent_type == 'folders': + if parent_id in resources['folders']: + return {'parent': f'{parent_type}/{parent_id}'} + if resources.get('organization') == parent_id: + return {'parent': f'{parent_type}/{parent_id}'} + + def _handle_discovery(resources, response, data): 'Processes the asset API response and returns parsed resources or next URL.' LOGGER.info('discovery handle request') @@ -73,6 +87,10 @@ def _handle_resource(resources, data): parent_data = _get_parent(data['parent'], resources) if not parent_data: LOGGER.info(f'{resource["self_link"]} outside perimeter') + LOGGER.debug([ + resources['organization'], resources['folders'], + resources['projects:number'] + ]) return resource.update(parent_data) # gets and calls the resource-level handler for type specific attributes @@ -186,20 +204,6 @@ def _self_link(s): return s.removeprefix('https://www.googleapis.com/compute/v1/') -def _get_parent(parent, resources): - 'Extracts and returns resource parent and type.' - parent_type, parent_id = parent.split('/')[-2:] - if parent_type == 'projects': - project = resources['projects:number'].get(parent_id) - if project: - return {'project_id': project['project_id'], 'project_number': parent_id} - if parent_type == 'folders': - if parent_id in resources['folders']: - return {'parent': f'{parent_type}/{parent_id}'} - if resources.get('organization') == parent_id: - return {'parent': f'{parent_type}/{parent_id}'} - - def _url(resources): 'Returns discovery URL' discovery_root = resources['config:discovery_root'] diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-metric-descriptors.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-metric-descriptors.py index 3af926be6e..a9e4090def 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-metric-descriptors.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-metric-descriptors.py @@ -54,7 +54,7 @@ def init(resources): resources.setdefault(NAME, {}) -@register_discovery(Level.CORE, 0) +@register_discovery(Level.CORE, 99) def start_discovery(resources, response=None, data=None): 'Plugin entry point, triggers discovery and handles requests and responses.' LOGGER.info(f'discovery (has response: {response is not None})') From 4cfd81086d4e849e25d076f711e0108aafe48ed6 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 08:29:21 +0100 Subject: [PATCH 66/82] cf deployment --- .../deploy-cloud-function/main.tf | 138 ++++++++++++++++++ .../deploy-cloud-function/outputs.tf | 0 .../deploy-cloud-function/variables.tf | 99 +++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf create mode 100644 blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf create mode 100644 blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf new file mode 100644 index 0000000000..d79cd770e1 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf @@ -0,0 +1,138 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + discovery_roles = ["roles/compute.viewer", "roles/cloudasset.viewer"] +} + +resource "random_string" "default" { + count = var.cloud_function_config.bucket_name == null ? 1 : 0 + length = 8 + special = false + upper = false +} + +module "project" { + source = "../../../../modules/project" + name = var.project_id + billing_account = try(var.project_create_config.billing_account_id, null) + labels = var.project_create_config != null ? var.labels : null + parent = try(var.project_create_config.parent_id, null) + project_create = var.project_create_config != null + services = [ + "cloudasset.googleapis.com", + "cloudbuild.googleapis.com", + "cloudfunctions.googleapis.com", + "cloudscheduler.googleapis.com", + "compute.googleapis.com", + "monitoring.googleapis.com" + ] +} + +module "pubsub" { + source = "../../../../modules/pubsub" + project_id = module.project.project_id + name = var.name + regions = [var.region] + subscriptions = { "${var.name}-default" = null } +} + +module "cloud-function" { + source = "../../../../modules/cloud-function" + project_id = module.project.project_id + name = var.name + bucket_name = coalesce( + var.cloud_function_config.bucket_name, + "${var.name}-${random_string.default.0.id}" + ) + bucket_config = { + location = var.region + } + build_worker_pool = var.cloud_function_config.build_worker_pool_id + bundle_config = { + source_dir = var.cloud_function_config.source_dir + output_path = var.cloud_function_config.bundle_path + } + environment_variables = ( + var.cloud_function_config.debug != true ? {} : { DEBUG = "1" } + ) + function_config = { + entry_point = "main_cf_pubsub" + memory_mb = var.cloud_function_config.memory_mb + timeout_seconds = var.cloud_function_config.timeout_seconds + } + service_account_create = true + trigger_config = { + v1 = { + event = "google.pubsub.topic.publish" + resource = module.pubsub.topic.id + } + } +} + +resource "google_cloud_scheduler_job" "job" { + project = var.project_id + region = var.region + name = var.name + schedule = var.schedule_config + time_zone = "UTC" + + pubsub_target { + attributes = {} + topic_name = module.pubsub.topic.id + data = base64encode(jsonencode({ + discovery_root = var.discovery_config.discovery_root + folders = var.discovery_config.monitored_folders + projects = var.discovery_config.monitored_projects + op_project = module.project.project_id + custom_quota = ( + var.discovery_config.custom_quota_file == null + ? { networks = {}, projects = {} } + : yamldecode(file(var.discovery_config.custom_quota_file)) + ) + })) + } +} + +resource "google_organization_iam_member" "discovery" { + for_each = toset( + var.grant_discovery_iam_roles && + startswith(var.discovery_config.discovery_root, "organizations/") + ? local.discovery_roles + : [] + ) + org_id = split("/", var.discovery_config.discovery_root)[1] + role = each.key + member = module.cloud-function.service_account_iam_email +} + +resource "google_folder_iam_member" "discovery" { + for_each = toset( + var.grant_discovery_iam_roles && + startswith(var.discovery_config.discovery_root, "folders/") + ? local.discovery_roles + : [] + ) + folder = var.discovery_config.discovery_root + role = each.key + member = module.cloud-function.service_account_iam_email +} + +resource "google_project_iam_member" "monitoring" { + project = module.project.project_id + role = "roles/monitoring.metricWriter" + member = module.cloud-function.service_account_iam_email +} diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf new file mode 100644 index 0000000000..6de0eb5d93 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "bundle_path" { + description = "Path used to write the intermediate Cloud Function code bundle." + type = string + default = "./bundle.zip" +} + +variable "cloud_function_config" { + description = "Optional Cloud Function configuration." + type = object({ + bucket_name = optional(string) + build_worker_pool_id = optional(string) + bundle_path = optional(string, "./bundle.zip") + debug = optional(bool, false) + memory_mb = optional(number, 256) + source_dir = optional(string, "../src") + timeout_seconds = optional(number, 540) + }) + default = {} + nullable = false +} + +variable "discovery_config" { + description = "Discovery configuration." + type = object({ + discovery_root = string + monitored_folders = list(string) + monitored_projects = list(string) + custom_quota_file = optional(string) + }) + nullable = false + validation { + condition = ( + var.discovery_config.monitored_folders != null && + var.discovery_config.monitored_projects != null + ) + error_message = "Monitored folders and projects can be empty lists, but they cannot be null." + } +} + +variable "grant_discovery_iam_roles" { + description = "Optionally grant required IAM roles to Cloud Function service account." + type = bool + default = false + nullable = false +} + +variable "labels" { + description = "Billing labels used for the Cloud Function, and the project if project_create is true." + type = map(string) + default = {} +} + +variable "name" { + description = "Name used to create Cloud Function related resources." + type = string + default = "net-dash" +} + +variable "project_create_config" { + description = "Optional configuration if project creation is required." + type = object({ + billing_account_id = string + parent_id = optional(string) + }) + default = null +} + +variable "project_id" { + description = "Project id where the Cloud Function will be deployed." + type = string +} + +variable "region" { + description = "Compute region where the Cloud Function will be deployed." + type = string + default = "europe-west1" +} + +variable "schedule_config" { + description = "Schedule timer configuration in crontab format." + type = string + default = "0/30 * * * *" +} From 6d6dabcbdbdb354e481d261802feaccf7881bb92 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 08:30:19 +0100 Subject: [PATCH 67/82] remove old paths --- .../network-dashboard/cloud-function/main.py | 239 ----------- .../cloud-function/metrics.yaml | 213 ---------- .../metrics/firewall_policies.py | 118 ------ .../cloud-function/metrics/ilb_fwrules.py | 122 ------ .../cloud-function/metrics/instances.py | 103 ----- .../cloud-function/metrics/limits.py | 236 ----------- .../cloud-function/metrics/metrics.py | 265 ------------- .../cloud-function/metrics/networks.py | 160 -------- .../cloud-function/metrics/peerings.py | 179 --------- .../cloud-function/metrics/routers.py | 57 --- .../cloud-function/metrics/routes.py | 289 -------------- .../cloud-function/metrics/subnets.py | 373 ------------------ .../cloud-function/metrics/vpc_firewalls.py | 122 ------ .../cloud-function/requirements.txt | 11 - .../network-dashboard/main.tf | 191 --------- .../network-dashboard/tests/README.md | 1 - .../network-dashboard/tests/test.tf | 287 -------------- .../network-dashboard/tests/variables.tf | 52 --- .../network-dashboard/variables.tf | 89 ----- .../network-dashboard/versions.tf | 27 -- 20 files changed, 3134 deletions(-) delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/main.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/networks.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routers.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/requirements.txt delete mode 100644 blueprints/cloud-operations/network-dashboard/main.tf delete mode 100644 blueprints/cloud-operations/network-dashboard/tests/README.md delete mode 100644 blueprints/cloud-operations/network-dashboard/tests/test.tf delete mode 100644 blueprints/cloud-operations/network-dashboard/tests/variables.tf delete mode 100644 blueprints/cloud-operations/network-dashboard/variables.tf delete mode 100644 blueprints/cloud-operations/network-dashboard/versions.tf diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py deleted file mode 100644 index 83e93fbb73..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ /dev/null @@ -1,239 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# CFv2 define whether to use Cloud function 2nd generation or 1st generation - -import re -from distutils.command.config import config -import os -import time -from google.cloud import monitoring_v3, asset_v1 -from google.protobuf import field_mask_pb2 -from googleapiclient import discovery -from metrics import ilb_fwrules, firewall_policies, instances, networks, metrics, limits, peerings, routes, subnets, vpc_firewalls - -CF_VERSION = os.environ.get("CF_VERSION") - - -def get_monitored_projects_list(config): - ''' - Gets the projects to be monitored from the MONITORED_FOLDERS_LIST environment variable. - - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - monitored_projects (List of strings): Full list of projects to be monitored - ''' - monitored_projects = config["monitored_projects"] - monitored_folders = os.environ.get("MONITORED_FOLDERS_LIST").split(",") - - # Handling empty monitored folders list - if monitored_folders == ['']: - monitored_folders = [] - - # Gets all projects under each monitored folder (and even in sub folders) - for folder in monitored_folders: - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"folders/{folder}", - "asset_types": ["cloudresourcemanager.googleapis.com/Project"], - "read_mask": read_mask - }) - - for resource in response: - for versioned in resource.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == "projectId": - project_id = field_value - # Avoid duplicate - if project_id not in monitored_projects: - monitored_projects.append(project_id) - - print("List of projects to be monitored:") - print(monitored_projects) - - return monitored_projects - - -def monitoring_interval(): - ''' - Creates the monitoring interval of 24 hours - Returns: - monitoring_v3.TimeInterval: Monitoring time interval of 24h - ''' - now = time.time() - seconds = int(now) - nanos = int((now - seconds) * 10**9) - return monitoring_v3.TimeInterval({ - "end_time": { - "seconds": seconds, - "nanos": nanos - }, - "start_time": { - "seconds": (seconds - 24 * 60 * 60), - "nanos": nanos - }, - }) - - -config = { - # Organization ID containing the projects to be monitored - "organization": - os.environ.get("ORGANIZATION_ID"), - # list of projects from which function will get quotas information - "monitored_projects": - os.environ.get("MONITORED_PROJECTS_LIST").split(","), - "monitoring_project": - os.environ.get('MONITORING_PROJECT_ID'), - "monitoring_project_link": - f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", - "monitoring_interval": - monitoring_interval(), - "limit_names": { - "GCE_INSTANCES": - "compute.googleapis.com/quota/instances_per_vpc_network/limit", - "L4": - "compute.googleapis.com/quota/internal_lb_forwarding_rules_per_vpc_network/limit", - "L7": - "compute.googleapis.com/quota/internal_managed_forwarding_rules_per_vpc_network/limit", - "SUBNET_RANGES": - "compute.googleapis.com/quota/subnet_ranges_per_vpc_network/limit" - }, - "lb_scheme": { - "L7": "INTERNAL_MANAGED", - "L4": "INTERNAL" - }, - "clients": { - "discovery_client": discovery.build('compute', 'v1'), - "asset_client": asset_v1.AssetServiceClient(), - "monitoring_client": monitoring_v3.MetricServiceClient() - }, - # Improve performance for Asset Inventory queries on large environments - "page_size": - 500, - "series_buffer": [], -} - - -def main(event, context=None): - ''' - Cloud Function Entry point, called by the scheduler. - Parameters: - event: Not used for now (Pubsub trigger) - context: Not used for now (Pubsub trigger) - Returns: - 'Function executed successfully' - ''' - # Handling empty monitored projects list - if config["monitored_projects"] == ['']: - config["monitored_projects"] = [] - - # Gets projects and folders to be monitored - config["monitored_projects"] = get_monitored_projects_list(config) - - # Keep the monitoring interval up2date during each run - config["monitoring_interval"] = monitoring_interval() - - metrics_dict, limits_dict = metrics.create_metrics( - config["monitoring_project_link"], config) - project_quotas_dict = limits.get_quota_project_limit(config) - - firewalls_dict = vpc_firewalls.get_firewalls_dict(config) - firewall_policies_dict = firewall_policies.get_firewall_policies_dict(config) - - # IP utilization subnet level metrics - subnets.get_subnets(config, metrics_dict) - - # Asset inventory queries - gce_instance_dict = instances.get_gce_instance_dict(config) - l4_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L4") - l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") - subnet_range_dict = networks.get_subnet_ranges_dict(config) - static_routes_dict = routes.get_static_routes_dict(config) - dynamic_routes_dict = routes.get_dynamic_routes( - config, metrics_dict, limits_dict['dynamic_routes_per_network_limit']) - - try: - - # Per Project metrics - vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, - firewalls_dict) - # Per Firewall Policy metrics - firewall_policies.get_firewal_policies_data(config, metrics_dict, - firewall_policies_dict) - # Per Network metrics - instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, - limits_dict['number_of_instances_limit']) - ilb_fwrules.get_forwarding_rules_data( - config, metrics_dict, l4_forwarding_rules_dict, - limits_dict['internal_forwarding_rules_l4_limit'], "L4") - ilb_fwrules.get_forwarding_rules_data( - config, metrics_dict, l7_forwarding_rules_dict, - limits_dict['internal_forwarding_rules_l7_limit'], "L7") - - routes.get_static_routes_data(config, metrics_dict, static_routes_dict, - project_quotas_dict) - - peerings.get_vpc_peering_data(config, metrics_dict, - limits_dict['number_of_vpc_peerings_limit']) - - # Per VPC peering group metrics - metrics.get_pgg_data( - config, - metrics_dict["metrics_per_peering_group"]["instance_per_peering_group"], - gce_instance_dict, config["limit_names"]["GCE_INSTANCES"], - limits_dict['number_of_instances_ppg_limit']) - metrics.get_pgg_data( - config, metrics_dict["metrics_per_peering_group"] - ["l4_forwarding_rules_per_peering_group"], l4_forwarding_rules_dict, - config["limit_names"]["L4"], - limits_dict['internal_forwarding_rules_l4_ppg_limit']) - metrics.get_pgg_data( - config, metrics_dict["metrics_per_peering_group"] - ["l7_forwarding_rules_per_peering_group"], l7_forwarding_rules_dict, - config["limit_names"]["L7"], - limits_dict['internal_forwarding_rules_l7_ppg_limit']) - metrics.get_pgg_data( - config, metrics_dict["metrics_per_peering_group"] - ["subnet_ranges_per_peering_group"], subnet_range_dict, - config["limit_names"]["SUBNET_RANGES"], - limits_dict['number_of_subnet_IP_ranges_ppg_limit']) - #static - routes.get_routes_ppg( - config, metrics_dict["metrics_per_peering_group"] - ["static_routes_per_peering_group"], static_routes_dict, - limits_dict['static_routes_per_peering_group_limit']) - #dynamic - routes.get_routes_ppg( - config, metrics_dict["metrics_per_peering_group"] - ["dynamic_routes_per_peering_group"], dynamic_routes_dict, - limits_dict['dynamic_routes_per_peering_group_limit']) - except Exception as e: - print("Error writing metrics") - print(e) - finally: - metrics.flush_series_buffer(config) - - return 'Function execution completed' - - -if CF_VERSION == "V2": - import functions_framework - main_http = functions_framework.http(main) - -if __name__ == "__main__": - main(None, None) \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml deleted file mode 100644 index 0c5bb8cc36..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ /dev/null @@ -1,213 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ---- -metrics_per_subnet: - ip_usage_per_subnet: - usage: - name: number_of_ip_used - description: Number of used IP addresses in the subnet. - utilization: - name: ip_addresses_per_subnet_utilization - description: Percentage of IP used in the subnet. - limit: - name: number_of_max_ip - description: Number of available IP addresses in the subnet. -metrics_per_network: - instance_per_network: - usage: - name: number_of_instances_usage - description: Number of instances per VPC network - usage. - limit: - name: number_of_instances_limit - description: Number of instances per VPC network - limit. - values: - default_value: 15000 - utilization: - name: number_of_instances_utilization - description: Number of instances per VPC network - utilization. - vpc_peering_active_per_network: - usage: - name: number_of_active_vpc_peerings_usage - description: Number of active VPC Peerings per VPC - usage. - limit: - name: number_of_active_vpc_peerings_limit - description: Number of active VPC Peerings per VPC - limit. - values: - default_value: 25 - utilization: - name: number_of_active_vpc_peerings_utilization - description: Number of active VPC Peerings per VPC - utilization. - vpc_peering_per_network: - usage: - name: number_of_vpc_peerings_usage - description: Number of VPC Peerings per VPC - usage. - limit: - name: number_of_vpc_peerings_limit - description: Number of VPC Peerings per VPC - limit. - values: - default_value: 25 - https://www.googleapis.com/compute/v1/projects/net-dash-test-host-prod/global/networks/vpc-prod: 40 - utilization: - name: number_of_vpc_peerings_utilization - description: Number of VPC Peerings per VPC - utilization. - l4_forwarding_rules_per_network: - usage: - name: internal_forwarding_rules_l4_usage - description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - usage. - limit: - name: internal_forwarding_rules_l4_limit - description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - limit. - values: - default_value: 500 - utilization: - name: internal_forwarding_rules_l4_utilization - description: Number of Internal Forwarding Rules for Internal L4 Load Balancers - utilization. - l7_forwarding_rules_per_network: - usage: - name: internal_forwarding_rules_l7_usage - description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per network - usage. - limit: - name: internal_forwarding_rules_l7_limit - description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per network - effective limit. - values: - default_value: 75 - utilization: - name: internal_forwarding_rules_l7_utilization - description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per Vnetwork - utilization. - dynamic_routes_per_network: - usage: - name: dynamic_routes_per_network_usage - description: Number of Dynamic routes per network - usage. - limit: - name: dynamic_routes_per_network_limit - description: Number of Dynamic routes per network - limit. - values: - default_value: 100 - utilization: - name: dynamic_routes_per_network_utilization - description: Number of Dynamic routes per network - utilization. - #static routes limit is per project, but usage is per network - static_routes_per_project: - usage: - name: static_routes_per_project_vpc_usage - description: Number of Static routes per project and network - usage. - limit: - name: static_routes_per_project_limit - description: Number of Static routes per project - limit. - values: - default_value: 250 - utilization: - name: static_routes_per_project_utilization - description: Number of Static routes per project - utilization. -metrics_per_peering_group: - l4_forwarding_rules_per_peering_group: - usage: - name: internal_forwarding_rules_l4_ppg_usage - description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - usage. - limit: - name: internal_forwarding_rules_l4_ppg_limit - description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - effective limit. - values: - default_value: 500 - utilization: - name: internal_forwarding_rules_l4_ppg_utilization - description: Number of Internal Forwarding Rules for Internal L4 Load Balancers per VPC peering group - utilization. - l7_forwarding_rules_per_peering_group: - usage: - name: internal_forwarding_rules_l7_ppg_usage - description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - usage. - limit: - name: internal_forwarding_rules_l7_ppg_limit - description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - effective limit. - values: - default_value: 175 - utilization: - name: internal_forwarding_rules_l7_ppg_utilization - description: Number of Internal Forwarding Rules for Internal L7 Load Balancers per VPC peering group - utilization. - subnet_ranges_per_peering_group: - usage: - name: number_of_subnet_IP_ranges_ppg_usage - description: Number of Subnet Ranges per peering group - usage. - limit: - name: number_of_subnet_IP_ranges_ppg_limit - description: Number of Subnet Ranges per peering group - effective limit. - values: - default_value: 400 - utilization: - name: number_of_subnet_IP_ranges_ppg_utilization - description: Number of Subnet Ranges per peering group - utilization. - instance_per_peering_group: - usage: - name: number_of_instances_ppg_usage - description: Number of instances per peering group - usage. - limit: - name: number_of_instances_ppg_limit - description: Number of instances per peering group - limit. - values: - default_value: 15500 - utilization: - name: number_of_instances_ppg_utilization - description: Number of instances per peering group - utilization. - dynamic_routes_per_peering_group: - usage: - name: dynamic_routes_per_peering_group_usage - description: Number of Dynamic routes per peering group - usage. - limit: - name: dynamic_routes_per_peering_group_limit - description: Number of Dynamic routes per peering group - limit. - values: - default_value: 300 - utilization: - name: dynamic_routes_per_peering_group_utilization - description: Number of Dynamic routes per peering group - utilization. - static_routes_per_peering_group: - usage: - name: static_routes_per_peering_group_usage - description: Number of Static routes per peering group - usage. - limit: - name: static_routes_per_peering_group_limit - description: Number of Static routes per peering group - limit. - values: - default_value: 300 - utilization: - name: static_routes_per_peering_group_utilization - description: Number of Static routes per peering group - utilization. -metrics_per_project: - firewalls: - usage: - name: firewalls_per_project_vpc_usage - description: Number of VPC firewall rules in a project - usage. - limit: - # Firewalls limit is per project and we get the limit for the GCP quota API in vpc_firewalls.py - name: firewalls_per_project_limit - description: Number of VPC firewall rules in a project - limit. - utilization: - name: firewalls_per_project_utilization - description: Number of VPC firewall rules in a project - utilization. -metrics_per_firewall_policy: - firewall_policy_tuples: - usage: - name: firewall_policy_tuples_per_policy_usage - description: Number of tuples in a firewall policy - usage. - limit: - # This limit is not visibile through Google APIs, set default_value - name: firewall_policy_tuples_per_policy_limit - description: Number of tuples in a firewall policy - limit. - values: - default_value: 2000 - utilization: - name: firewall_policy_tuples_per_policy_utilization - description: Number of tuples in a firewall policy - utilization. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py deleted file mode 100644 index 95a26db383..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/firewall_policies.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import re -import time - -from collections import defaultdict -from pydoc import doc -from collections import defaultdict -from google.protobuf import field_mask_pb2 -from . import metrics, networks, limits - - -def get_firewall_policies_dict(config: dict): - ''' - Calls the Asset Inventory API to get all Firewall Policies under the GCP organization, including children - Ignores monitored projects list: returns all policies regardless of their parent resource - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - firewal_policies_dict (dictionary of dictionary): Keys are policy ids, subkeys are policy field values - ''' - - firewall_policies_dict = defaultdict(int) - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/FirewallPolicy"], - "read_mask": read_mask, - }) - for resource in response: - for versioned in resource.versioned_resources: - firewall_policy = dict() - for field_name, field_value in versioned.resource.items(): - firewall_policy[field_name] = field_value - firewall_policies_dict[firewall_policy['id']] = firewall_policy - return firewall_policies_dict - - -def get_firewal_policies_data(config, metrics_dict, firewall_policies_dict): - ''' - Gets the data for VPC Firewall Policies in an organization, including children. All folders are considered, - only projects in the monitored projects list are considered. - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. - firewall_policies_dict (dictionary of of dictionary of string: string): Keys are policies ids, subkeys are policies values - Returns: - None - ''' - - current_tuples_limit = None - try: - current_tuples_limit = metrics_dict["metrics_per_firewall_policy"][ - "firewall_policy_tuples"]["limit"]["values"]["default_value"] - except Exception: - print( - f"Could not determine number of tuples metric limit due to missing default value" - ) - if current_tuples_limit < 0: - print( - f"Could not determine number of tuples metric limit as default value is <= 0" - ) - - timestamp = time.time() - for firewall_policy_key in firewall_policies_dict: - firewall_policy = firewall_policies_dict[firewall_policy_key] - - # may either be a org, a folder, or a project - # folder and org require to split {folder,organization}\/\w+ - parent = re.search("(\w+$)", firewall_policy["parent"]).group( - 1) if "parent" in firewall_policy else re.search( - "([\d,a-z,-]+)(\/[\d,a-z,-]+\/firewallPolicies/[\d,a-z,-]*$)", - firewall_policy["selfLink"]).group(1) - parent_type = re.search("(^\w+)", firewall_policy["parent"]).group( - 1) if "parent" in firewall_policy else "projects" - - if parent_type == "projects" and parent not in config["monitored_projects"]: - continue - - metric_labels = {'parent': parent, 'parent_type': parent_type} - - metric_labels["name"] = firewall_policy[ - "displayName"] if "displayName" in firewall_policy else firewall_policy[ - "name"] - - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_firewall_policy"] - [f"firewall_policy_tuples"]["usage"]["name"], - firewall_policy['ruleTupleCount'], metric_labels, timestamp=timestamp) - if not current_tuples_limit == None and current_tuples_limit > 0: - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_firewall_policy"] - [f"firewall_policy_tuples"]["limit"]["name"], current_tuples_limit, - metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_firewall_policy"] - [f"firewall_policy_tuples"]["utilization"]["name"], - firewall_policy['ruleTupleCount'] / current_tuples_limit, - metric_labels, timestamp=timestamp) - - print(f"Buffered number tuples per Firewall Policy") diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py deleted file mode 100644 index de8274d973..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/ilb_fwrules.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from collections import defaultdict -from google.protobuf import field_mask_pb2 -from . import metrics, networks, limits - - -def get_forwarding_rules_dict(config, layer: str): - ''' - Calls the Asset Inventory API to get all L4 Forwarding Rules under the GCP organization. - - Parameters: - config (dict): The dict containing config like clients and limits - layer (string): the Layer to get Forwarding rules (L4/L7) - Returns: - forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network. - ''' - - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - forwarding_rules_dict = defaultdict(int) - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/ForwardingRule"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - - for resource in response: - internal = False - network_link = "" - for versioned in resource.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == "loadBalancingScheme": - internal = (field_value == config["lb_scheme"][layer]) - if field_name == "network": - network_link = field_value - if internal: - if network_link in forwarding_rules_dict: - forwarding_rules_dict[network_link] += 1 - else: - forwarding_rules_dict[network_link] = 1 - - return forwarding_rules_dict - - -def get_forwarding_rules_data(config, metrics_dict, forwarding_rules_dict, - limit_dict, layer): - ''' - Gets the data for L4 Internal Forwarding Rules per VPC Network and writes it to the metric defined in forwarding_rules_metric. - - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. - forwarding_rules_dict (dictionary of string: int): Keys are the network links and values are the number of Forwarding Rules per network. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. - layer (string): the Layer to get Forwarding rules (L4/L7) - Returns: - None - ''' - - timestamp = time.time() - for project_id in config["monitored_projects"]: - network_dict = networks.get_networks(config, project_id) - - current_quota_limit = limits.get_quota_current_limit( - config, f"projects/{project_id}", config["limit_names"][layer]) - - if current_quota_limit is None: - print( - f"Could not determine {layer} forwarding rules to metric for projects/{project_id} due to missing quotas" - ) - continue - - current_quota_limit_view = metrics.customize_quota_view(current_quota_limit) - - for net in network_dict: - limits.set_limits(net, current_quota_limit_view, limit_dict) - - usage = 0 - if net['self_link'] in forwarding_rules_dict: - usage = forwarding_rules_dict[net['self_link']] - - metric_labels = { - 'project': project_id, - 'network_name': net['network_name'] - } - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - [f"{layer.lower()}_forwarding_rules_per_network"]["usage"]["name"], - usage, metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - [f"{layer.lower()}_forwarding_rules_per_network"]["limit"]["name"], - net['limit'], metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - [f"{layer.lower()}_forwarding_rules_per_network"]["utilization"] - ["name"], usage / net['limit'], metric_labels, timestamp=timestamp) - - print( - f"Buffered number of {layer} forwarding rules to metric for projects/{project_id}" - ) \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py deleted file mode 100644 index d3b72e678e..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/instances.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from code import interact -from collections import defaultdict -from . import metrics, networks, limits - - -def get_gce_instance_dict(config: dict): - ''' - Calls the Asset Inventory API to get all GCE instances under the GCP organization. - - Parameters: - config (dict): The dict containing config like clients and limits - - Returns: - gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network. - ''' - - gce_instance_dict = defaultdict(int) - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Instance"], - "page_size": config["page_size"], - }) - for resource in response: - for field_name, field_value in resource.additional_attributes.items(): - if field_name == "networkInterfaceNetworks": - for network in field_value: - if network in gce_instance_dict: - gce_instance_dict[network] += 1 - else: - gce_instance_dict[network] = 1 - - return gce_instance_dict - - -def get_gce_instances_data(config, metrics_dict, gce_instance_dict, limit_dict): - ''' - Gets the data for GCE instances per VPC Network and writes it to the metric defined in instance_metric. - - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions - gce_instance_dict (dictionary of string: int): Keys are the network links and values are the number of GCE Instances per network. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - gce_instance_dict - ''' - timestamp = time.time() - for project_id in config["monitored_projects"]: - network_dict = networks.get_networks(config, project_id) - - current_quota_limit = limits.get_quota_current_limit( - config, f"projects/{project_id}", - config["limit_names"]["GCE_INSTANCES"]) - if current_quota_limit is None: - print( - f"Could not determine number of instances for projects/{project_id} due to missing quotas" - ) - - current_quota_limit_view = metrics.customize_quota_view(current_quota_limit) - - for net in network_dict: - limits.set_limits(net, current_quota_limit_view, limit_dict) - - usage = 0 - if net['self_link'] in gce_instance_dict: - usage = gce_instance_dict[net['self_link']] - - metric_labels = { - 'project': project_id, - 'network_name': net['network_name'] - } - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["instance_per_network"] - ["usage"]["name"], usage, metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["instance_per_network"] - ["limit"]["name"], net['limit'], metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["instance_per_network"] - ["utilization"]["name"], usage / net['limit'], metric_labels, - timestamp=timestamp) - - print(f"Buffered number of instances to metric for projects/{project_id}") diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py deleted file mode 100644 index edd4a50b3d..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ /dev/null @@ -1,236 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from google.api_core import exceptions -from google.cloud import monitoring_v3 -from . import metrics - - -def get_quotas_dict(quotas_list): - ''' - Creates a dictionary of quotas from a list, with lower case quota name as keys - Parameters: - quotas_array (array): array of quotas - Returns: - quotas_dict (dict): dictionary of quotas - ''' - quota_keys = [q['metric'] for q in quotas_list] - quotas_dict = dict() - i = 0 - for key in quota_keys: - if ("metric" in quotas_list[i]): - del (quotas_list[i]["metric"]) - quotas_dict[key.lower()] = quotas_list[i] - i += 1 - return quotas_dict - - -def get_quota_project_limit(config, regions=["global"]): - ''' - Retrieves quotas for all monitored project in selected regions, default 'global' - Parameters: - project_link (string): Project link. - Returns: - quotas (dict): quotas for all selected regions, default 'global' - ''' - try: - request = {} - quotas = dict() - for project in config["monitored_projects"]: - quotas[project] = dict() - if regions != ["global"]: - for region in regions: - request = config["clients"]["discovery_client"].compute.regions().get( - region=region, project=project) - response = request.execute() - quotas[project][region] = get_quotas_dict(response['quotas']) - else: - region = "global" - request = config["clients"]["discovery_client"].projects().get( - project=project, fields="quotas") - response = request.execute() - quotas[project][region] = get_quotas_dict(response['quotas']) - - return quotas - except exceptions.PermissionDenied as err: - print( - f"Warning: error reading quotas for {project}. " + - f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" - ) - return None - - -def get_ppg(network_link, limit_dict): - ''' - Checks if this network has a specific limit for a metric, if so, returns that limit, if not, returns the default limit. - - Parameters: - network_link (string): VPC network link. - limit_list (list of string): Used to get the limit per VPC or the default limit. - Returns: - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - ''' - if network_link in limit_dict: - return limit_dict[network_link] - else: - if 'default_value' in limit_dict: - return limit_dict['default_value'] - else: - print(f"Error: limit not found for {network_link}") - return 0 - - -def set_limits(network_dict, quota_limit, limit_dict): - ''' - Updates the network dictionary with quota limit values. - - Parameters: - network_dict (dictionary of string: string): Contains network information. - quota_limit (list of dictionaries of string: string): Current quota limit. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - None - ''' - - network_dict['limit'] = None - - if quota_limit: - for net in quota_limit: - if net['network_id'] == network_dict['network_id']: - network_dict['limit'] = net['value'] - return - - network_link = f"https://www.googleapis.com/compute/v1/projects/{network_dict['project_id']}/global/networks/{network_dict['network_name']}" - - if network_link in limit_dict: - network_dict['limit'] = limit_dict[network_link] - else: - if 'default_value' in limit_dict: - network_dict['limit'] = limit_dict['default_value'] - else: - print(f"Error: Couldn't find limit for {network_link}") - network_dict['limit'] = 0 - - -def get_quota_current_limit(config, project_link, metric_name): - ''' - Retrieves limit for a specific metric. - - Parameters: - project_link (string): Project link. - metric_name (string): Name of the metric. - Returns: - results_list (list of string): Current limit. - ''' - - try: - results = config["clients"]["monitoring_client"].list_time_series( - request={ - "name": project_link, - "filter": f'metric.type = "{metric_name}"', - "interval": config["monitoring_interval"], - "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL - }) - results_list = list(results) - return results_list - except exceptions.PermissionDenied as err: - print( - f"Warning: error reading quotas for {project_link}. " + - f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" - ) - return None - - -def count_effective_limit(config, project_id, network_dict, usage_metric_name, - limit_metric_name, utilization_metric_name, - limit_dict, timestamp=None): - ''' - Calculates the effective limits (using algorithm in the link below) for peering groups and writes data (usage, limit, utilization) to the custom metrics. - Source: https://cloud.google.com/vpc/docs/quota#vpc-peering-effective-limit - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project to be analyzed. - network_dict (dictionary of string: string): Contains all required information about the network to get the usage, limit and utilization. - usage_metric_name (string): Name of the custom metric to be populated for usage per VPC peering group. - limit_metric_name (string): Name of the custom metric to be populated for limit per VPC peering group. - utilization_metric_name (string): Name of the custom metric to be populated for utilization per VPC peering group. - limit_dict (dictionary of string:int): Dictionary containing the limit per peering group (either VPC specific or default limit). - timestamp (time): timestamp to be recorded for all points - Returns: - None - ''' - - if timestamp == None: - timestamp = time.time() - - if network_dict['peerings'] == []: - return - - # Get usage: Sums usage for current network + all peered networks - peering_group_usage = network_dict['usage'] - for peered_network in network_dict['peerings']: - if 'usage' not in peered_network: - print( - f"Cannot add metrics for peered network in projects/{project_id} as no usage metrics exist due to missing permissions" - ) - continue - peering_group_usage += peered_network['usage'] - - network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" - - # Calculates effective limit: Step 1: max(per network limit, per network_peering_group limit) - limit_step1 = max(network_dict['limit'], get_ppg(network_link, limit_dict)) - - # Calculates effective limit: Step 2: List of max(per network limit, per network_peering_group limit) for each peered network - limit_step2 = [] - for peered_network in network_dict['peerings']: - peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network['project_id']}/global/networks/{peered_network['network_name']}" - - if 'limit' in peered_network: - limit_step2.append( - max(peered_network['limit'], get_ppg(peered_network_link, - limit_dict))) - else: - print( - f"Ignoring projects/{peered_network['project_id']} for limits in peering group of project {project_id} as no limits are available." - + - "This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" - ) - - # Calculates effective limit: Step 3: Find minimum from the list created by Step 2 - limit_step3 = 0 - if len(limit_step2) > 0: - limit_step3 = min(limit_step2) - - # Calculates effective limit: Step 4: Find maximum from step 1 and step 3 - effective_limit = max(limit_step1, limit_step3) - utilization = peering_group_usage / effective_limit - metric_labels = { - 'project': project_id, - 'network_name': network_dict['network_name'] - } - metrics.append_data_to_series_buffer(config, usage_metric_name, - peering_group_usage, metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer(config, limit_metric_name, - effective_limit, metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer(config, utilization_metric_name, - utilization, metric_labels, - timestamp=timestamp) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py deleted file mode 100644 index b26e30d454..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ /dev/null @@ -1,265 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from curses import KEY_MARK -import re -import time -import yaml -from google.api import metric_pb2 as ga_metric -from google.cloud import monitoring_v3 -from . import peerings, limits, networks - - -def create_metrics(monitoring_project, config): - ''' - Creates all Cloud Monitoring custom metrics based on the metric.yaml file - Parameters: - monitoring_project (string): the project where the metrics are written to - config (dict): The dict containing config like clients and limits - Returns: - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions - limits_dict (dictionary of dictionary of string: int): limits_dict[metric_name]: dict[network_name] = limit_value - ''' - client = config["clients"]["monitoring_client"] - existing_metrics = [] - for desc in client.list_metric_descriptors(name=monitoring_project): - existing_metrics.append(desc.type) - limits_dict = {} - - with open("./metrics.yaml", 'r') as stream: - try: - metrics_dict = yaml.safe_load(stream) - - for metric_list in metrics_dict.values(): - for metric_name, metric in metric_list.items(): - for sub_metric_key, sub_metric in metric.items(): - metric_link = f"custom.googleapis.com/{sub_metric['name']}" - # If the metric doesn't exist yet, then we create it - if metric_link not in existing_metrics: - create_metric(sub_metric["name"], sub_metric["description"], - monitoring_project, config) - # Parse limits for network and peering group metrics - # Subnet level metrics have a different limit: the subnet IP range size - if sub_metric_key == "limit" and metric_name != "ip_usage_per_subnet": - limits_dict_for_metric = {} - if "values" in sub_metric: - for network_link, limit_value in sub_metric["values"].items(): - limits_dict_for_metric[network_link] = limit_value - limits_dict[sub_metric["name"]] = limits_dict_for_metric - - return metrics_dict, limits_dict - except yaml.YAMLError as exc: - print(exc) - - -def create_metric(metric_name, description, monitoring_project, config): - ''' - Creates a Cloud Monitoring metric based on the parameter given if the metric is not already existing - Parameters: - metric_name (string): Name of the metric to be created - description (string): Description of the metric to be created - monitoring_project (string): the project where the metrics are written to - config (dict): The dict containing config like clients and limits - Returns: - None - ''' - client = config["clients"]["monitoring_client"] - - descriptor = ga_metric.MetricDescriptor() - descriptor.type = f"custom.googleapis.com/{metric_name}" - descriptor.metric_kind = ga_metric.MetricDescriptor.MetricKind.GAUGE - descriptor.value_type = ga_metric.MetricDescriptor.ValueType.DOUBLE - descriptor.description = description - descriptor = client.create_metric_descriptor(name=monitoring_project, - metric_descriptor=descriptor) - print("Created {}.".format(descriptor.name)) - - -def append_data_to_series_buffer(config, metric_name, metric_value, - metric_labels, timestamp=None): - ''' - Appends data to Cloud Monitoring custom metrics, using a buffer. buffer is flushed every BUFFER_LEN elements, - any unflushed series is discarded upon function closure - Parameters: - config (dict): The dict containing config like clients and limits - metric_name (string): Name of the metric - metric_value (int): Value for the data point of the metric. - matric_labels (dictionary of dictionary of string: string): metric labels names and values - timestamp (float): seconds since the epoch, in UTC - Returns: - usage (int): Current usage for that network. - limit (int): Current usage for that network. - ''' - - # Configurable buffer size to improve performance when writing datapoints to metrics - buffer_len = 10 - - series = monitoring_v3.TimeSeries() - series.metric.type = f"custom.googleapis.com/{metric_name}" - series.resource.type = "global" - - for label_name in metric_labels: - if (metric_labels[label_name] != None): - series.metric.labels[label_name] = metric_labels[label_name] - - timestamp = timestamp if timestamp != None else time.time() - seconds = int(timestamp) - nanos = int((timestamp - seconds) * 10**9) - interval = monitoring_v3.TimeInterval( - {"end_time": { - "seconds": seconds, - "nanos": nanos - }}) - point = monitoring_v3.Point({ - "interval": interval, - "value": { - "double_value": metric_value - } - }) - series.points = [point] - - # TODO: sometimes this cashes with 'DeadlineExceeded: 504 Deadline expired before operation could complete' error - # Implement exponential backoff retries? - config["series_buffer"].append(series) - if len(config["series_buffer"]) >= buffer_len: - flush_series_buffer(config) - - -def flush_series_buffer(config): - ''' - writes buffered metrics to Google Cloud Monitoring, empties buffer upon both failure/success - config (dict): The dict containing config like clients and limits - ''' - try: - if config["series_buffer"] and len(config["series_buffer"]) > 0: - client = config["clients"]["monitoring_client"] - client.create_time_series(name=config["monitoring_project_link"], - time_series=config["series_buffer"]) - series_names = [ - re.search("\/(.+$)", series.metric.type).group(1) - for series in config["series_buffer"] - ] - print("Wrote time series: ", series_names) - except Exception as e: - print("Error while flushing series buffer") - print(e) - - config["series_buffer"] = [] - - -def get_pgg_data(config, metric_dict, usage_dict, limit_metric, limit_dict): - ''' - This function gets the usage, limit and utilization per VPC peering group for a specific metric for all projects to be monitored. - Parameters: - config (dict): The dict containing config like clients and limits - metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics - usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value - limit_metric (string): Name of the existing GCP metric for limit per VPC network - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - None - ''' - for project_id in config["monitored_projects"]: - network_dict_list = peerings.gather_peering_data(config, project_id) - # Network dict list is a list of dictionary (one for each network) - # For each network, this dictionary contains: - # project_id, network_name, network_id, usage, limit, peerings (list of peered networks) - # peerings is a list of dictionary (one for each peered network) and contains: - # project_id, network_name, network_id - current_quota_limit = limits.get_quota_current_limit( - config, f"projects/{project_id}", limit_metric) - if current_quota_limit is None: - print( - f"Could not determine number of L7 forwarding rules to metric for projects/{project_id} due to missing quotas" - ) - continue - - current_quota_limit_view = customize_quota_view(current_quota_limit) - - timestamp = time.time() - # For each network in this GCP project - for network_dict in network_dict_list: - if network_dict['network_id'] == 0: - print( - f"Could not determine {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id} due to missing permissions." - ) - continue - network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" - - limit = networks.get_limit_network(network_dict, network_link, - current_quota_limit_view, limit_dict) - - usage = 0 - if network_link in usage_dict: - usage = usage_dict[network_link] - - # Here we add usage and limit to the network dictionary - network_dict["usage"] = usage - network_dict["limit"] = limit - - # For every peered network, get usage and limits - for peered_network_dict in network_dict['peerings']: - peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}" - peered_usage = 0 - if peered_network_link in usage_dict: - peered_usage = usage_dict[peered_network_link] - - current_peered_quota_limit = limits.get_quota_current_limit( - config, f"projects/{peered_network_dict['project_id']}", - limit_metric) - if current_peered_quota_limit is None: - print( - f"Could not determine metrics for peering to projects/{peered_network_dict['project_id']} due to missing quotas" - ) - continue - - peering_project_limit = customize_quota_view(current_peered_quota_limit) - - peered_limit = networks.get_limit_network(peered_network_dict, - peered_network_link, - peering_project_limit, - limit_dict) - # Here we add usage and limit to the peered network dictionary - peered_network_dict["usage"] = peered_usage - peered_network_dict["limit"] = peered_limit - - limits.count_effective_limit(config, project_id, network_dict, - metric_dict["usage"]["name"], - metric_dict["limit"]["name"], - metric_dict["utilization"]["name"], - limit_dict, timestamp) - print( - f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" - ) - - -def customize_quota_view(quota_results): - ''' - Customize the quota output for an easier parsable output. - Parameters: - quota_results (string): Input from get_quota_current_usage or get_quota_current_limit. Contains the Current usage or limit for all networks in that project. - Returns: - quotaViewList (list of dictionaries of string: string): Current quota usage or limit. - ''' - quotaViewList = [] - for result in quota_results: - quotaViewJson = {} - quotaViewJson.update(dict(result.resource.labels)) - quotaViewJson.update(dict(result.metric.labels)) - for val in result.points: - quotaViewJson.update({'value': val.value.int64_value}) - quotaViewList.append(quotaViewJson) - return quotaViewList \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/networks.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/networks.py deleted file mode 100644 index 094f374ed6..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/networks.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from code import interact -from collections import defaultdict -from google.protobuf import field_mask_pb2 -from googleapiclient import errors -import http - - -def get_subnet_ranges_dict(config: dict): - ''' - Calls the Asset Inventory API to get all Subnet ranges under the GCP organization. - - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - subnet_range_dict (dictionary of string: int): Keys are the network links and values are the number of subnet ranges per network. - ''' - - subnet_range_dict = defaultdict(int) - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Subnetwork"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - for resource in response: - ranges = 0 - network_link = None - - for versioned in resource.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == "network": - network_link = field_value - ranges += 1 - if field_name == "secondaryIpRanges": - for range in field_value: - ranges += 1 - - if network_link in subnet_range_dict: - subnet_range_dict[network_link] += ranges - else: - subnet_range_dict[network_link] = ranges - - return subnet_range_dict - - -def get_networks(config, project_id): - ''' - Returns a dictionary of all networks in a project. - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project containing the networks. - Returns: - network_dict (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) - ''' - request = config["clients"]["discovery_client"].networks().list( - project=project_id) - response = request.execute() - network_dict = [] - if 'items' in response: - for network in response['items']: - network_name = network['name'] - network_id = network['id'] - self_link = network['selfLink'] - d = { - 'project_id': project_id, - 'network_name': network_name, - 'network_id': network_id, - 'self_link': self_link - } - network_dict.append(d) - return network_dict - - -def get_network_id(config, project_id, network_name): - ''' - Returns the network_id for a specific project / network name. - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project containing the networks. - network_name (string): Name of the network - Returns: - network_id (int): Network ID. - ''' - request = config["clients"]["discovery_client"].networks().list( - project=project_id) - try: - response = request.execute() - except errors.HttpError as err: - # TODO: log proper warning - if err.resp.status == http.HTTPStatus.FORBIDDEN: - print( - f"Warning: error reading networks for {project_id}. " + - f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" - ) - else: - print(f"Warning: error reading networks for {project_id}: {err}") - return 0 - - network_id = 0 - - if 'items' in response: - for network in response['items']: - if network['name'] == network_name: - network_id = network['id'] - break - - if network_id == 0: - print(f"Error: network_id not found for {network_name} in {project_id}") - - return network_id - - -def get_limit_network(network_dict, network_link, quota_limit, limit_dict): - ''' - Returns limit for a specific network and metric, using the GCP quota metrics or the values in the yaml file if not found. - - Parameters: - network_dict (dictionary of string: string): Contains network information. - network_link (string): Contains network link - quota_limit (list of dictionaries of string: string): Current quota limit for all networks in that project. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - limit (int): Current limit for that network. - ''' - if quota_limit: - for net in quota_limit: - if net['network_id'] == network_dict['network_id']: - return net['value'] - - if network_link in limit_dict: - return limit_dict[network_link] - else: - if 'default_value' in limit_dict: - return limit_dict['default_value'] - else: - print(f"Error: Couldn't find limit for {network_link}") - - return 0 diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py deleted file mode 100644 index 616c7f6630..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/peerings.py +++ /dev/null @@ -1,179 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from . import metrics, networks, limits - - -def get_vpc_peering_data(config, metrics_dict, limit_dict): - ''' - Gets the data for VPC peerings (active or not) and writes it to the metric defined (vpc_peering_active_metric and vpc_peering_metric). - - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - None - ''' - timestamp = time.time() - for project in config["monitored_projects"]: - active_vpc_peerings, vpc_peerings = gather_vpc_peerings_data( - config, project, limit_dict) - - for peering in active_vpc_peerings: - metric_labels = { - 'project': project, - 'network_name': peering['network_name'] - } - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["vpc_peering_active_per_network"]["usage"]["name"], - peering['active_peerings'], metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["vpc_peering_active_per_network"]["limit"]["name"], - peering['network_limit'], metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["vpc_peering_active_per_network"]["utilization"]["name"], - peering['active_peerings'] / peering['network_limit'], metric_labels, - timestamp=timestamp) - print( - "Buffered number of active VPC peerings to custom metric for project:", - project) - - for peering in vpc_peerings: - metric_labels = { - 'project': project, - 'network_name': peering['network_name'] - } - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["vpc_peering_per_network"] - ["usage"]["name"], peering['peerings'], metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["vpc_peering_per_network"] - ["limit"]["name"], peering['network_limit'], metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["vpc_peering_per_network"] - ["utilization"]["name"], - peering['peerings'] / peering['network_limit'], metric_labels, - timestamp=timestamp) - print("Buffered number of VPC peerings to custom metric for project:", - project) - - -def gather_peering_data(config, project_id): - ''' - Returns a dictionary of all peerings for all networks in a project. - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project containing the networks. - Returns: - network_list (dictionary of string: string): Contains the project_id, network_name(s) and network_id(s) of peered networks. - ''' - request = config["clients"]["discovery_client"].networks().list( - project=project_id) - response = request.execute() - - network_list = [] - if 'items' in response: - for network in response['items']: - net = { - 'project_id': project_id, - 'network_name': network['name'], - 'network_id': network['id'], - 'peerings': [] - } - if 'peerings' in network: - STATE = network['peerings'][0]['state'] - if STATE == "ACTIVE": - for peered_network in network[ - 'peerings']: # "projects/{project_name}/global/networks/{network_name}" - start = peered_network['network'].find("projects/") + len( - 'projects/') - end = peered_network['network'].find("/global") - peered_project = peered_network['network'][start:end] - peered_network_name = peered_network['network'].split( - "networks/")[1] - peered_net = { - 'project_id': - peered_project, - 'network_name': - peered_network_name, - 'network_id': - networks.get_network_id(config, peered_project, - peered_network_name) - } - net["peerings"].append(peered_net) - network_list.append(net) - return network_list - - -def gather_vpc_peerings_data(config, project_id, limit_dict): - ''' - Gets the data for all VPC peerings (active or not) in project_id and writes it to the metric defined in vpc_peering_active_metric and vpc_peering_metric. - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): We will take all VPCs in that project_id and look for all peerings to these VPCs. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - active_peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each active VPC peering. - peerings_dict (dictionary of string: string): Contains project_id, network_name, network_limit for each VPC peering. - ''' - active_peerings_dict = [] - peerings_dict = [] - request = config["clients"]["discovery_client"].networks().list( - project=project_id) - response = request.execute() - if 'items' in response: - for network in response['items']: - if 'peerings' in network: - STATE = network['peerings'][0]['state'] - if STATE == "ACTIVE": - active_peerings_count = len(network['peerings']) - else: - active_peerings_count = 0 - - peerings_count = len(network['peerings']) - else: - peerings_count = 0 - active_peerings_count = 0 - - network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network['name']}" - network_limit = limits.get_ppg(network_link, limit_dict) - - active_d = { - 'project_id': project_id, - 'network_name': network['name'], - 'active_peerings': active_peerings_count, - 'network_limit': network_limit - } - active_peerings_dict.append(active_d) - d = { - 'project_id': project_id, - 'network_name': network['name'], - 'peerings': peerings_count, - 'network_limit': network_limit - } - peerings_dict.append(d) - - return active_peerings_dict, peerings_dict diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routers.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routers.py deleted file mode 100644 index 064354e7f6..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routers.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from google.protobuf import field_mask_pb2 - - -def get_routers(config): - ''' - Returns a dictionary of all Cloud Routers in the GCP organization. - - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - routers_dict (dictionary of string: list of string): Key is the network link and value is a list of router links. - ''' - - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - routers_dict = {} - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Router"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - for resource in response: - network_link = None - router_link = None - for versioned in resource.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == "network": - network_link = field_value - if field_name == "selfLink": - router_link = field_value - - if network_link in routers_dict: - routers_dict[network_link].append(router_link) - else: - routers_dict[network_link] = [router_link] - - return routers_dict diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py deleted file mode 100644 index a161454547..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/routes.py +++ /dev/null @@ -1,289 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from collections import defaultdict -from google.protobuf import field_mask_pb2 -from . import metrics, networks, limits, peerings, routers - - -def get_routes_for_router(config, project_id, router_region, router_name): - ''' - Returns the same of dynamic routes learned by a specific Cloud Router instance - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project containing the Cloud Router. - router_region (string): GCP region for the Cloud Router. - router_name (string): Cloud Router name. - Returns: - sum_routes (int): Number of dynamic routes learned by the Cloud Router. - ''' - request = config["clients"]["discovery_client"].routers().getRouterStatus( - project=project_id, region=router_region, router=router_name) - response = request.execute() - - sum_routes = 0 - - if 'result' in response: - if 'bgpPeerStatus' in response['result']: - for peer in response['result']['bgpPeerStatus']: - sum_routes += peer['numLearnedRoutes'] - - return sum_routes - - -def get_routes_for_network(config, network_link, project_id, routers_dict): - ''' - Returns a the number of dynamic routes for a given network - - Parameters: - config (dict): The dict containing config like clients and limits - network_link (string): Network self link. - project_id (string): Project ID containing the network. - routers_dict (dictionary of string: list of string): Dictionary with key as network link and value as list of router links. - Returns: - sum_routes (int): Number of routes in that network. - ''' - sum_routes = 0 - - if network_link in routers_dict: - for router_link in routers_dict[network_link]: - # Router link is using the following format: - # 'https://www.googleapis.com/compute/v1/projects/PROJECT_ID/regions/REGION/routers/ROUTER_NAME' - start = router_link.find("/regions/") + len("/regions/") - end = router_link.find("/routers/") - router_region = router_link[start:end] - router_name = router_link.split('/routers/')[1] - routes = get_routes_for_router(config, project_id, router_region, - router_name) - - sum_routes += routes - - return sum_routes - - -def get_dynamic_routes(config, metrics_dict, limits_dict): - ''' - This function gets the usage, limit and utilization for the dynamic routes per VPC - note: assumes global routing is ON for all VPCs - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. - limits_dict (dictionary of string: int): key is network link (or 'default_value') and value is the limit for that network - Returns: - dynamic_routes_dict (dictionary of string: int): key is network link and value is the number of dynamic routes for that network - ''' - routers_dict = routers.get_routers(config) - dynamic_routes_dict = defaultdict(int) - - timestamp = time.time() - for project in config["monitored_projects"]: - network_dict = networks.get_networks(config, project) - - for net in network_dict: - sum_routes = get_routes_for_network(config, net['self_link'], project, - routers_dict) - dynamic_routes_dict[net['self_link']] = sum_routes - - if net['self_link'] in limits_dict: - limit = limits_dict[net['self_link']] - else: - if 'default_value' in limits_dict: - limit = limits_dict['default_value'] - else: - print("Error: couldn't find limit for dynamic routes.") - break - - utilization = sum_routes / limit - metric_labels = {'project': project, 'network_name': net['network_name']} - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["dynamic_routes_per_network"]["usage"]["name"], sum_routes, - metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["dynamic_routes_per_network"]["limit"]["name"], limit, metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"] - ["dynamic_routes_per_network"]["utilization"]["name"], utilization, - metric_labels, timestamp=timestamp) - - print("Buffered metrics for dynamic routes for VPCs in project", project) - - return dynamic_routes_dict - - -def get_routes_ppg(config, metric_dict, usage_dict, limit_dict): - ''' - This function gets the usage, limit and utilization for the static or dynamic routes per VPC peering group. - note: assumes global routing is ON for all VPCs for dynamic routes, assumes share custom routes is on for all peered networks - Parameters: - config (dict): The dict containing config like clients and limits - metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics - usage_dict (dictionnary of string:int): Dictionary with the network link as key and the number of resources as value - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value - Returns: - None - ''' - timestamp = time.time() - for project_id in config["monitored_projects"]: - network_dict_list = peerings.gather_peering_data(config, project_id) - - for network_dict in network_dict_list: - network_link = f"https://www.googleapis.com/compute/v1/projects/{project_id}/global/networks/{network_dict['network_name']}" - - limit = limits.get_ppg(network_link, limit_dict) - - usage = 0 - if network_link in usage_dict: - usage = usage_dict[network_link] - - # Here we add usage and limit to the network dictionary - network_dict["usage"] = usage - network_dict["limit"] = limit - - # For every peered network, get usage and limits - for peered_network_dict in network_dict['peerings']: - peered_network_link = f"https://www.googleapis.com/compute/v1/projects/{peered_network_dict['project_id']}/global/networks/{peered_network_dict['network_name']}" - peered_usage = 0 - if peered_network_link in usage_dict: - peered_usage = usage_dict[peered_network_link] - - peered_limit = limits.get_ppg(peered_network_link, limit_dict) - - # Here we add usage and limit to the peered network dictionary - peered_network_dict["usage"] = peered_usage - peered_network_dict["limit"] = peered_limit - - limits.count_effective_limit(config, project_id, network_dict, - metric_dict["usage"]["name"], - metric_dict["limit"]["name"], - metric_dict["utilization"]["name"], - limit_dict, timestamp) - print( - f"Buffered {metric_dict['usage']['name']} for peering group {network_dict['network_name']} in {project_id}" - ) - - -def get_static_routes_dict(config): - ''' - Calls the Asset Inventory API to get all static custom routes under the GCP organization. - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - routes_per_vpc_dict (dictionary of string: int): Keys are the network links and values are the number of custom static routes per network. - ''' - routes_per_vpc_dict = defaultdict() - usage_dict = defaultdict() - - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Route"], - "read_mask": read_mask - }) - - for resource in response: - for versioned in resource.versioned_resources: - static_route = dict() - for field_name, field_value in versioned.resource.items(): - static_route[field_name] = field_value - static_route["project_id"] = static_route["network"].split('/')[6] - static_route["network_name"] = static_route["network"].split('/')[-1] - network_link = f"https://www.googleapis.com/compute/v1/projects/{static_route['project_id']}/global/networks/{static_route['network_name']}" - #exclude default vpc and peering routes, dynamic routes are not in Cloud Asset Inventory - if "nextHopPeering" not in static_route and "nextHopNetwork" not in static_route: - if network_link not in routes_per_vpc_dict: - routes_per_vpc_dict[network_link] = dict() - routes_per_vpc_dict[network_link]["project_id"] = static_route[ - "project_id"] - routes_per_vpc_dict[network_link]["network_name"] = static_route[ - "network_name"] - if static_route["destRange"] not in routes_per_vpc_dict[network_link]: - routes_per_vpc_dict[network_link][static_route["destRange"]] = {} - if "usage" not in routes_per_vpc_dict[network_link]: - routes_per_vpc_dict[network_link]["usage"] = 0 - routes_per_vpc_dict[network_link][ - "usage"] = routes_per_vpc_dict[network_link]["usage"] + 1 - - #output a dict with network links and usage only - return { - network_link_out: routes_per_vpc_dict[network_link_out]["usage"] - for network_link_out in routes_per_vpc_dict - } - - -def get_static_routes_data(config, metrics_dict, static_routes_dict, - project_quotas_dict): - ''' - Determines and writes the number of static routes for each VPC in monitored projects, the per project limit and the per project utilization - note: assumes custom routes sharing is ON for all VPCs - Parameters: - config (dict): The dict containing config like clients and limits - metric_dict (dictionary of string: string): Dictionary with the metric names and description, that will be used to populate the metrics - static_routes_dict (dictionary of dictionary: int): Keys are the network links and values are the number of custom static routes per network. - project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. - Returns: - None - ''' - timestamp = time.time() - project_usage = {project: 0 for project in config["monitored_projects"]} - - #usage is drilled down by network - for network_link in static_routes_dict: - - project_id = network_link.split('/')[6] - if (project_id not in config["monitored_projects"]): - continue - network_name = network_link.split('/')[-1] - - project_usage[project_id] = project_usage[project_id] + static_routes_dict[ - network_link] - - metric_labels = {"project": project_id, "network_name": network_name} - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["static_routes_per_project"] - ["usage"]["name"], static_routes_dict[network_link], metric_labels, - timestamp=timestamp) - - #limit and utilization are calculated by project - for project_id in project_usage: - current_quota_limit = project_quotas_dict[project_id]['global']["routes"][ - "limit"] - if current_quota_limit is None: - print( - f"Could not determine static routes metric for projects/{project_id} due to missing quotas" - ) - continue - # limit and utilization are calculted by project - metric_labels = {"project": project_id} - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["static_routes_per_project"] - ["limit"]["name"], current_quota_limit, metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_network"]["static_routes_per_project"] - ["utilization"]["name"], - project_usage[project_id] / current_quota_limit, metric_labels, - timestamp=timestamp) - - return diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py deleted file mode 100644 index 46fbc7564a..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/subnets.py +++ /dev/null @@ -1,373 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from . import metrics -from google.protobuf import field_mask_pb2 -from google.protobuf.json_format import MessageToDict -import ipaddress - - -def get_all_subnets(config): - ''' - Returns a dictionary with subnet level informations (such as IP utilization) - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - subnet_dict (dictionary of String: dictionary): Key is the project_id, value is a nested dictionary with subnet_region/subnet_name as the key. - ''' - subnet_dict = {} - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ['compute.googleapis.com/Subnetwork'], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - - for asset in response: - for versioned in asset.versioned_resources: - subnet_name = "" - network_name = "" - project_id = "" - ip_cidr_range = "" - subnet_region = "" - - for field_name, field_value in versioned.resource.items(): - if field_name == 'name': - subnet_name = field_value - elif field_name == 'network': - # Network self link format: - # "https://www.googleapis.com/compute/v1/projects//global/networks/" - project_id = field_value.split('/')[6] - network_name = field_value.split('/')[-1] - elif field_name == 'ipCidrRange': - ip_cidr_range = field_value - elif field_name == 'region': - subnet_region = field_value.split('/')[-1] - - net = ipaddress.ip_network(ip_cidr_range) - # Note that 4 IP addresses are reserved by GCP in all subnets - # Source: https://cloud.google.com/vpc/docs/subnets#reserved_ip_addresses_in_every_subnet - total_ip_addresses = int(net.num_addresses) - 4 - - if project_id not in subnet_dict: - subnet_dict[project_id] = {} - subnet_dict[project_id][f"{subnet_region}/{subnet_name}"] = { - 'name': subnet_name, - 'region': subnet_region, - 'ip_cidr_range': ip_cidr_range, - 'total_ip_addresses': total_ip_addresses, - 'used_ip_addresses': 0, - 'network_name': network_name - } - - return subnet_dict - - -def compute_subnet_utilization_vms(config, read_mask, all_subnets_dict): - ''' - Counts VMs using private IPs in the different subnets. - Parameters: - config (dict): Dict containing config like clients and limits - read_mask (FieldMask): read_mask to get additional metadata from Cloud Asset Inventory - all_subnets_dict (dict): Dict containing the information for each subnets in the GCP organization - Returns: - None - ''' - response_vm = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Instance"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - - # Counting IP addresses for GCE instances (VMs) - for asset in response_vm: - for versioned in asset.versioned_resources: - for field_name, field_value in versioned.resource.items(): - # TODO: Handle multi-NIC - if field_name == 'networkInterfaces': - response_dict = MessageToDict(list(field_value._pb)[0]) - # Subnet self link: - # https://www.googleapis.com/compute/v1/projects//regions//subnetworks/ - subnet_region = response_dict['subnetwork'].split('/')[-3] - subnet_name = response_dict['subnetwork'].split('/')[-1] - # Network self link: - # https://www.googleapis.com/compute/v1/projects//global/networks/ - project_id = response_dict['network'].split('/')[6] - network_name = response_dict['network'].split('/')[-1] - - all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][ - 'used_ip_addresses'] += 1 - - -def compute_subnet_utilization_ilbs(config, read_mask, all_subnets_dict): - ''' - Counts ILBs using private IPs in the different subnets. - Parameters: - config (dict): Dict containing config like clients and limits - read_mask (FieldMask): read_mask to get additional metadata from Cloud Asset Inventory - all_subnets_dict (dict): Dict containing the information for each subnets in the GCP organization - Returns: - None - ''' - response_ilb = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/ForwardingRule"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - - for asset in response_ilb: - internal = False - psc = False - project_id = '' - subnet_name = '' - subnet_region = '' - address = '' - network = '' - for versioned in asset.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if 'loadBalancingScheme' in field_name and field_value in [ - 'INTERNAL', 'INTERNAL_MANAGED' - ]: - internal = True - # We want to count only accepted PSC endpoint Forwarding Rule - # If the PSC endpoint Forwarding Rule is pending, we will count it in the reserved IP addresses - elif field_name == 'pscConnectionStatus' and field_value == 'ACCEPTED': - psc = True - elif field_name == 'IPAddress': - address = field_value - elif field_name == 'network': - project_id = field_value.split('/')[6] - network = field_value.split('/')[-1] - elif 'subnetwork' in field_name: - subnet_name = field_value.split('/')[-1] - subnet_region = field_value.split('/')[-3] - - if internal: - all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][ - 'used_ip_addresses'] += 1 - elif psc: - # PSC endpoint asset doesn't contain the subnet information in Asset Inventory - # We need to find the correct subnet with IP address matching - ip_address = ipaddress.ip_address(address) - for subnet_key, subnet_dict in all_subnets_dict[project_id].items(): - if subnet_dict["network_name"] == network: - if ip_address in ipaddress.ip_network(subnet_dict['ip_cidr_range']): - all_subnets_dict[project_id][subnet_key]['used_ip_addresses'] += 1 - - -def compute_subnet_utilization_addresses(config, read_mask, all_subnets_dict): - ''' - Counts reserved IP addresses in the different subnets. - Parameters: - config (dict): Dict containing config like clients and limits - read_mask (FieldMask): read_mask to get additional metadata from Cloud Asset Inventory - all_subnets_dict (dict): Dict containing the information for each subnets in the GCP organization - Returns: - None - ''' - response_reserved_ips = config["clients"][ - "asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Address"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - - # Counting IP addresses for GCE Reserved IPs (ex: PSC, Cloud DNS Inbound policies, reserved GCE IPs) - for asset in response_reserved_ips: - purpose = "" - status = "" - project_id = "" - network_name = "" - subnet_name = "" - subnet_region = "" - address = "" - prefixLength = "" - address_name = "" - for versioned in asset.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == 'name': - address_name = field_value - if field_name == 'purpose': - purpose = field_value - elif field_name == 'region': - subnet_region = field_value.split('/')[-1] - elif field_name == 'status': - status = field_value - elif field_name == 'address': - address = field_value - elif field_name == 'network': - network_name = field_value.split('/')[-1] - project_id = field_value.split('/')[6] - elif field_name == 'subnetwork': - subnet_name = field_value.split('/')[-1] - project_id = field_value.split('/')[6] - elif field_name == 'prefixLength': - prefixLength = field_value - - # Rserved IP addresses for GCE instances or PSC Forwarding Rule PENDING state - if purpose == "GCE_ENDPOINT" and status == "RESERVED": - all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][ - 'used_ip_addresses'] += 1 - # Cloud DNS inbound policy - elif purpose == "DNS_RESOLVER": - all_subnets_dict[project_id][f"{subnet_region}/{subnet_name}"][ - 'used_ip_addresses'] += 1 - # PSA Range for Cloud SQL, MemoryStore, etc. - elif purpose == "VPC_PEERING": - ip_range = f"{address}/{int(prefixLength)}" - net = ipaddress.ip_network(ip_range) - # Note that 4 IP addresses are reserved by GCP in all subnets - # Source: https://cloud.google.com/vpc/docs/subnets#reserved_ip_addresses_in_every_subnet - total_ip_addresses = int(net.num_addresses) - 4 - all_subnets_dict[project_id][f"psa/{address_name}"] = { - 'name': f"psa/{address_name}", - 'region': subnet_region, - 'ip_cidr_range': ip_range, - 'total_ip_addresses': total_ip_addresses, - 'used_ip_addresses': 0, - 'network_name': network_name - } - - -def compute_subnet_utilization_redis(config, read_mask, all_subnets_dict): - ''' - Counts Redis (Memorystore) instances using private IPs in the different subnets. - Parameters: - config (dict): Dict containing config like clients and limits - read_mask (FieldMask): read_mask to get additional metadata from Cloud Asset Inventory - all_subnets_dict (dict): Dict containing the information for each subnets in the GCP organization - Returns: - None - ''' - response_redis = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["redis.googleapis.com/Instance"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - - for asset in response_redis: - ip_range = "" - connect_mode = "" - network_name = "" - project_id = "" - region = "" - for versioned in asset.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == 'locationId': - region = field_value[0:-2] - if field_name == 'authorizedNetwork': - network_name = field_value.split('/')[-1] - project_id = field_value.split('/')[1] - if field_name == 'reservedIpRange': - ip_range = field_value - if field_name == 'connectMode': - connect_mode = field_value - - # Only handling PSA for Redis for now - if connect_mode == "PRIVATE_SERVICE_ACCESS": - redis_ip_range = ipaddress.ip_network(ip_range) - for subnet_key, subnet_dict in all_subnets_dict[project_id].items(): - if subnet_dict["network_name"] == network_name: - # Reddis instance asset doesn't contain the subnet information in Asset Inventory - # We need to find the correct subnet range with IP address matching to compute the utilization - if redis_ip_range.overlaps( - ipaddress.ip_network(subnet_dict['ip_cidr_range'])): - all_subnets_dict[project_id][subnet_key][ - 'used_ip_addresses'] += redis_ip_range.num_addresses - all_subnets_dict[project_id][subnet_key]['region'] = region - - -def compute_subnet_utilization(config, all_subnets_dict): - ''' - Counts resources (VMs, ILBs, reserved IPs) using private IPs in the different subnets. - Parameters: - config (dict): Dict containing config like clients and limits - all_subnets_dict (dict): Dict containing the information for each subnets in the GCP organization - Returns: - None - ''' - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - compute_subnet_utilization_vms(config, read_mask, all_subnets_dict) - compute_subnet_utilization_ilbs(config, read_mask, all_subnets_dict) - compute_subnet_utilization_addresses(config, read_mask, all_subnets_dict) - # TODO: Other PSA services such as FileStore, Cloud SQL - compute_subnet_utilization_redis(config, read_mask, all_subnets_dict) - - # TODO: Handle secondary ranges and count GKE pods - - -def get_subnets(config, metrics_dict): - ''' - Writes all subnet metrics to custom metrics. - - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - None - ''' - - all_subnets_dict = get_all_subnets(config) - # Updates all_subnets_dict with the IP utilization info - compute_subnet_utilization(config, all_subnets_dict) - - timestamp = time.time() - for project_id in config["monitored_projects"]: - if project_id not in all_subnets_dict: - continue - for subnet_dict in all_subnets_dict[project_id].values(): - ip_utilization = 0 - if subnet_dict['used_ip_addresses'] > 0: - ip_utilization = subnet_dict['used_ip_addresses'] / subnet_dict[ - 'total_ip_addresses'] - - # Building unique identifier with subnet region/name - subnet_id = f"{subnet_dict['region']}/{subnet_dict['name']}" - metric_labels = { - 'project': project_id, - 'network_name': subnet_dict['network_name'], - 'subnet_id': subnet_id - } - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"] - ["usage"]["name"], subnet_dict['used_ip_addresses'], metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"] - ["limit"]["name"], subnet_dict['total_ip_addresses'], metric_labels, - timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_subnet"]["ip_usage_per_subnet"] - ["utilization"]["name"], ip_utilization, metric_labels, - timestamp=timestamp) - - print("Buffered metrics for subnet ip utilization for VPCs in project", - project_id) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py deleted file mode 100644 index f9fec79a72..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import re -import time - -from collections import defaultdict -from pydoc import doc -from collections import defaultdict -from google.protobuf import field_mask_pb2 -from . import metrics, networks, limits - - -def get_firewalls_dict(config: dict): - ''' - Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization. - - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules - ''' - - firewalls_dict = defaultdict(int) - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Firewall"], - "read_mask": read_mask, - "page_size": config["page_size"], - }) - for resource in response: - project_id = re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", - resource.name).group(2) - network_name = "" - for versioned in resource.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == "network": - network_name = re.search("[a-z0-9\-]*$", field_value).group(0) - firewalls_dict[project_id] = defaultdict( - int - ) if not project_id in firewalls_dict else firewalls_dict[project_id] - firewalls_dict[project_id][ - network_name] = 1 if not network_name in firewalls_dict[ - project_id] else firewalls_dict[project_id][network_name] + 1 - break - break - return firewalls_dict - - -def get_firewalls_data(config, metrics_dict, project_quotas_dict, - firewalls_dict): - ''' - Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric. - - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. - project_quotas_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. - firewalls_dict (dictionary of of dictionary of string: string): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules - Returns: - None - ''' - - timestamp = time.time() - for project_id in config["monitored_projects"]: - - current_quota_limit = project_quotas_dict[project_id]['global']["firewalls"] - if current_quota_limit is None: - print( - f"Could not determine VPC firewal rules metric for projects/{project_id} due to missing quotas" - ) - continue - - network_dict = networks.get_networks(config, project_id) - - project_usage = 0 - for net in network_dict: - usage = 0 - if project_id in firewalls_dict and net['network_name'] in firewalls_dict[ - project_id]: - usage = firewalls_dict[project_id][net['network_name']] - project_usage += usage - metric_labels = { - 'project': project_id, - 'network_name': net['network_name'] - } - metrics.append_data_to_series_buffer( - config, - metrics_dict["metrics_per_project"][f"firewalls"]["usage"]["name"], - usage, metric_labels, timestamp=timestamp) - - metric_labels = {'project': project_id} - # firewall quotas are per project, not per single VPC - metrics.append_data_to_series_buffer( - config, - metrics_dict["metrics_per_project"][f"firewalls"]["limit"]["name"], - current_quota_limit['limit'], metric_labels, timestamp=timestamp) - metrics.append_data_to_series_buffer( - config, metrics_dict["metrics_per_project"][f"firewalls"]["utilization"] - ["name"], project_usage / current_quota_limit['limit'] - if current_quota_limit['limit'] != 0 else 0, metric_labels, - timestamp=timestamp) - print( - f"Buffered number of VPC Firewall Rules to metric for projects/{project_id}" - ) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/requirements.txt b/blueprints/cloud-operations/network-dashboard/cloud-function/requirements.txt deleted file mode 100644 index d561348229..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -regex==2022.3.2 -google-api-python-client==2.39.0 -google-auth==2.6.0 -google-auth-httplib2==0.1.0 -google-cloud-logging==3.0.0 -google-cloud-monitoring==2.9.1 -oauth2client==4.1.3 -google-api-core==2.7.0 -PyYAML==6.0 -google-cloud-asset==3.8.1 -functions-framework==3.* \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/main.tf b/blueprints/cloud-operations/network-dashboard/main.tf deleted file mode 100644 index e74cabd6e2..0000000000 --- a/blueprints/cloud-operations/network-dashboard/main.tf +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - project_ids = toset(var.monitored_projects_list) - projects = join(",", local.project_ids) - - folder_ids = toset(var.monitored_folders_list) - folders = join(",", local.folder_ids) - monitoring_project = var.monitoring_project_id == "" ? module.project-monitoring[0].project_id : var.monitoring_project_id -} - -################################################ -# Monitoring project creation # -################################################ - -module "project-monitoring" { - count = var.monitoring_project_id == "" ? 1 : 0 - source = "../../../modules/project" - name = "network-dashboards" - parent = "organizations/${var.organization_id}" - prefix = var.prefix - billing_account = var.billing_account - services = var.project_monitoring_services -} - -################################################ -# Service account creation and IAM permissions # -################################################ - -module "service-account-function" { - source = "../../../modules/iam-service-account" - project_id = local.monitoring_project - name = "sa-dash" - generate_key = false - - # Required IAM permissions for this service account are: - # 1) compute.networkViewer on projects to be monitored (I gave it at organization level for now for simplicity) - # 2) monitoring viewer on the projects to be monitored (I gave it at organization level for now for simplicity) - - iam_organization_roles = { - "${var.organization_id}" = [ - "roles/compute.networkViewer", - "roles/monitoring.viewer", - "roles/cloudasset.viewer" - ] - } - - iam_project_roles = { - "${local.monitoring_project}" = [ - "roles/monitoring.metricWriter", - ] - } -} - -module "service-account-scheduler" { - source = "../../../modules/iam-service-account" - project_id = local.monitoring_project - name = "sa-scheduler" - generate_key = false - - iam_project_roles = { - "${local.monitoring_project}" = [ - "roles/run.invoker", - "roles/cloudfunctions.invoker" - ] - } -} - -################################################ -# Cloud Function configuration (& Scheduler) # -# you can comment out the pub/sub call in case of 2nd generation function -################################################ - -module "pubsub" { - - source = "../../../modules/pubsub" - project_id = local.monitoring_project - name = "network-dashboard-pubsub" - subscriptions = { - "network-dashboard-pubsub-default" = null - } - # the Cloud Scheduler robot service account already has pubsub.topics.publish - # at the project level via roles/cloudscheduler.serviceAgent -} - -resource "google_cloud_scheduler_job" "job" { - count = var.cf_version == "V2" ? 0 : 1 - project = local.monitoring_project - region = var.region - name = "network-dashboard-scheduler" - schedule = var.schedule_cron - time_zone = "UTC" - - pubsub_target { - topic_name = module.pubsub.topic.id - data = base64encode("test") - } -} -#http trigger for 2nd generation function - -resource "google_cloud_scheduler_job" "job_httptrigger" { - count = var.cf_version == "V2" ? 1 : 0 - project = local.monitoring_project - region = var.region - name = "network-dashboard-scheduler" - schedule = var.schedule_cron - time_zone = "UTC" - - http_target { - http_method = "POST" - uri = module.cloud-function.uri - - oidc_token { - service_account_email = module.service-account-scheduler.email - } - } -} - -module "cloud-function" { - v2 = var.cf_version == "V2" - source = "../../../modules/cloud-function" - project_id = local.monitoring_project - name = "network-dashboard-cloud-function" - bucket_name = "${local.monitoring_project}-network-dashboard-bucket" - bucket_config = { - location = var.region - } - region = var.region - - bundle_config = { - source_dir = "cloud-function" - output_path = "cloud-function.zip" - } - - function_config = { - timeout = 480 # Timeout in seconds, increase it if your CF timeouts and use v2 if > 9 minutes. - entry_point = "main" - runtime = "python39" - instances = 1 - memory_mb = 256 - - } - - environment_variables = { - MONITORED_PROJECTS_LIST = local.projects - MONITORED_FOLDERS_LIST = local.folders - MONITORING_PROJECT_ID = local.monitoring_project - ORGANIZATION_ID = var.organization_id - CF_VERSION = var.cf_version - } - - service_account = module.service-account-function.email - # Internal only doesn't seem to work with CFv2: - ingress_settings = var.cf_version == "V2" ? "ALLOW_ALL" : "ALLOW_INTERNAL_ONLY" - - trigger_config = var.cf_version == "V2" ? { - v2 = { - event_type = "google.cloud.pubsub.topic.v1.messagePublished" - pubsub_topic = module.pubsub.topic.id - service_account_create = true - } - } : { - v1 = { - event = "google.pubsub.topic.publish" - resource = module.pubsub.topic.id - } - } -} - -################################################ -# Cloud Monitoring Dashboard creation # -################################################ - -resource "google_monitoring_dashboard" "dashboard" { - dashboard_json = file("${path.module}/dashboards/quotas-utilization.json") - project = local.monitoring_project -} diff --git a/blueprints/cloud-operations/network-dashboard/tests/README.md b/blueprints/cloud-operations/network-dashboard/tests/README.md deleted file mode 100644 index 6e4779d459..0000000000 --- a/blueprints/cloud-operations/network-dashboard/tests/README.md +++ /dev/null @@ -1 +0,0 @@ -Creating here resources to test the Cloud Function and ensuring metrics are correctly populated \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/tests/test.tf b/blueprints/cloud-operations/network-dashboard/tests/test.tf deleted file mode 100644 index bb9d6d317c..0000000000 --- a/blueprints/cloud-operations/network-dashboard/tests/test.tf +++ /dev/null @@ -1,287 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_folder" "test-net-dash" { - display_name = "test-net-dash" - parent = "organizations/${var.organization_id}" -} - -##### Creating host projects, VPCs, service projects ##### - -module "project-hub" { - source = "../../../../modules/project" - name = "test-host-hub" - parent = google_folder.test-net-dash.name - prefix = var.prefix - billing_account = var.billing_account - services = var.project_vm_services - - shared_vpc_host_config = { - enabled = true - } -} - -module "vpc-hub" { - source = "../../../../modules/net-vpc" - project_id = module.project-hub.project_id - name = "vpc-hub" - subnets = [ - { - ip_cidr_range = "10.0.10.0/24" - name = "subnet-hub-1" - region = var.region - } - ] -} - -module "project-svc-hub" { - source = "../../../../modules/project" - parent = google_folder.test-net-dash.name - billing_account = var.billing_account - prefix = var.prefix - name = "test-svc-hub" - services = var.project_vm_services - - shared_vpc_service_config = { - attach = true - host_project = module.project-hub.project_id - } -} - -module "project-prod" { - source = "../../../../modules/project" - name = "test-host-prod" - parent = google_folder.test-net-dash.name - prefix = var.prefix - billing_account = var.billing_account - services = var.project_vm_services - - shared_vpc_host_config = { - enabled = true - } -} - -module "vpc-prod" { - source = "../../../../modules/net-vpc" - project_id = module.project-prod.project_id - name = "vpc-prod" - subnets = [ - { - ip_cidr_range = "10.0.20.0/24" - name = "subnet-prod-1" - region = var.region - } - ] -} - -module "project-svc-prod" { - source = "../../../../modules/project" - parent = google_folder.test-net-dash.name - billing_account = var.billing_account - prefix = var.prefix - name = "test-svc-prod" - services = var.project_vm_services - - shared_vpc_service_config = { - attach = true - host_project = module.project-prod.project_id - } -} - -module "project-dev" { - source = "../../../../modules/project" - name = "test-host-dev" - parent = google_folder.test-net-dash.name - prefix = var.prefix - billing_account = var.billing_account - services = var.project_vm_services - - shared_vpc_host_config = { - enabled = true - } -} - -module "vpc-dev" { - source = "../../../../modules/net-vpc" - project_id = module.project-dev.project_id - name = "vpc-dev" - subnets = [ - { - ip_cidr_range = "10.0.30.0/24" - name = "subnet-dev-1" - region = var.region - } - ] -} - -module "project-svc-dev" { - source = "../../../../modules/project" - parent = google_folder.test-net-dash.name - billing_account = var.billing_account - prefix = var.prefix - name = "test-svc-dev" - services = var.project_vm_services - - shared_vpc_service_config = { - attach = true - host_project = module.project-dev.project_id - } -} - -##### Creating VPC peerings ##### - -module "hub-to-prod-peering" { - source = "../../../../modules/net-vpc-peering" - local_network = module.vpc-hub.self_link - peer_network = module.vpc-prod.self_link -} - -module "prod-to-hub-peering" { - source = "../../../../modules/net-vpc-peering" - local_network = module.vpc-prod.self_link - peer_network = module.vpc-hub.self_link - depends_on = [module.hub-to-prod-peering] -} - -module "hub-to-dev-peering" { - source = "../../../../modules/net-vpc-peering" - local_network = module.vpc-hub.self_link - peer_network = module.vpc-dev.self_link -} - -module "dev-to-hub-peering" { - source = "../../../../modules/net-vpc-peering" - local_network = module.vpc-dev.self_link - peer_network = module.vpc-hub.self_link - depends_on = [module.hub-to-dev-peering] -} - -##### Creating VMs ##### - -resource "google_compute_instance" "test-vm-prod1" { - project = module.project-svc-prod.project_id - name = "test-vm-prod1" - machine_type = "f1-micro" - zone = var.zone - - tags = ["${var.region}"] - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - network_interface { - subnetwork = module.vpc-prod.subnet_self_links["${var.region}/subnet-prod-1"] - subnetwork_project = module.project-prod.project_id - } - - allow_stopping_for_update = true -} - -resource "google_compute_instance" "test-vm-prod2" { - project = module.project-prod.project_id - name = "test-vm-prod2" - machine_type = "f1-micro" - zone = var.zone - - tags = [var.region] - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - network_interface { - subnetwork = module.vpc-prod.subnet_self_links["${var.region}/subnet-prod-1"] - subnetwork_project = module.project-prod.project_id - } - - allow_stopping_for_update = true -} - -resource "google_compute_instance" "test-vm-dev1" { - count = 10 - project = module.project-svc-dev.project_id - name = "test-vm-dev${count.index}" - machine_type = "f1-micro" - zone = var.zone - - tags = ["${var.region}"] - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - network_interface { - subnetwork = module.vpc-dev.subnet_self_links["${var.region}/subnet-dev-1"] - subnetwork_project = module.project-dev.project_id - } - - allow_stopping_for_update = true -} - -resource "google_compute_instance" "test-vm-hub1" { - project = module.project-svc-hub.project_id - name = "test-vm-hub1" - machine_type = "f1-micro" - zone = var.zone - - tags = ["${var.region}"] - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - network_interface { - subnetwork = module.vpc-hub.subnet_self_links["${var.region}/subnet-hub-1"] - subnetwork_project = module.project-hub.project_id - } - - allow_stopping_for_update = true -} - -# Forwarding Rules -resource "google_compute_forwarding_rule" "forwarding-rule-dev" { - count = 10 - name = "forwarding-rule-dev${count.index}" - project = module.project-svc-dev.project_id - network = module.vpc-dev.self_link - subnetwork = module.vpc-dev.subnet_self_links["${var.region}/subnet-dev-1"] - - region = var.region - backend_service = google_compute_region_backend_service.test-backend.id - ip_protocol = "TCP" - load_balancing_scheme = "INTERNAL" - all_ports = true - allow_global_access = true - -} - -# backend service -resource "google_compute_region_backend_service" "test-backend" { - name = "test-backend" - region = var.region - project = module.project-svc-dev.project_id - protocol = "TCP" - load_balancing_scheme = "INTERNAL" -} diff --git a/blueprints/cloud-operations/network-dashboard/tests/variables.tf b/blueprints/cloud-operations/network-dashboard/tests/variables.tf deleted file mode 100644 index dd01b29fdf..0000000000 --- a/blueprints/cloud-operations/network-dashboard/tests/variables.tf +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "organization_id" { - description = "The organization id for the associated services" -} - -variable "billing_account" { - description = "The ID of the billing account to associate this project with" -} - -variable "prefix" { - description = "Prefix used for resource names." - type = string - validation { - condition = var.prefix != "" - error_message = "Prefix cannot be empty." - } -} - -variable "project_vm_services" { - description = "Service APIs enabled by default in new projects." - default = [ - "cloudbilling.googleapis.com", - "compute.googleapis.com", - "logging.googleapis.com", - "monitoring.googleapis.com", - "servicenetworking.googleapis.com", - ] -} -variable "region" { - description = "Region used to deploy subnets" - default = "europe-west1" -} - -variable "zone" { - description = "Zone used to deploy vms" - default = "europe-west1-b" -} diff --git a/blueprints/cloud-operations/network-dashboard/variables.tf b/blueprints/cloud-operations/network-dashboard/variables.tf deleted file mode 100644 index 2744eed629..0000000000 --- a/blueprints/cloud-operations/network-dashboard/variables.tf +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "billing_account" { - description = "The ID of the billing account to associate this project with." -} - -variable "cf_version" { - description = "Cloud Function version 2nd Gen or 1st Gen. Possible options: 'V1' or 'V2'.Use CFv2 if your Cloud Function timeouts after 9 minutes. By default it is using CFv1." - default = "V1" - validation { - condition = var.cf_version == "V1" || var.cf_version == "V2" - error_message = "The value of cf_version must be either V1 or V2." - } -} - -variable "monitored_folders_list" { - type = list(string) - description = "ID of the projects to be monitored (where limits and quotas data will be pulled)." - default = [] -} - -variable "monitored_projects_list" { - type = list(string) - description = "ID of the projects to be monitored (where limits and quotas data will be pulled)." -} - -variable "monitoring_project_id" { - description = "Monitoring project where the dashboard will be created and the solution deployed; a project will be created if set to empty string." - default = "" -} - -variable "organization_id" { - description = "The organization id for the associated services." -} - -variable "prefix" { - description = "Prefix used for resource names." - type = string - validation { - condition = var.prefix != "" - error_message = "Prefix cannot be empty." - } -} - -variable "project_monitoring_services" { - description = "Service APIs enabled in the monitoring project if it will be created." - default = [ - "artifactregistry.googleapis.com", - "cloudasset.googleapis.com", - "cloudbilling.googleapis.com", - "cloudbuild.googleapis.com", - "cloudfunctions.googleapis.com", - "cloudresourcemanager.googleapis.com", - "cloudscheduler.googleapis.com", - "compute.googleapis.com", - "iam.googleapis.com", - "iamcredentials.googleapis.com", - "logging.googleapis.com", - "monitoring.googleapis.com", - "pubsub.googleapis.com", - "run.googleapis.com", - "servicenetworking.googleapis.com", - "serviceusage.googleapis.com", - "storage-component.googleapis.com" - ] -} -variable "region" { - description = "Region used to deploy the cloud functions and scheduler." - default = "europe-west1" -} - -variable "schedule_cron" { - description = "Cron format schedule to run the Cloud Function. Default is every 10 minutes." - default = "*/10 * * * *" -} diff --git a/blueprints/cloud-operations/network-dashboard/versions.tf b/blueprints/cloud-operations/network-dashboard/versions.tf deleted file mode 100644 index 3bdf23370a..0000000000 --- a/blueprints/cloud-operations/network-dashboard/versions.tf +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -terraform { - required_version = ">= 1.3.1" - required_providers { - google = { - source = "hashicorp/google" - version = ">= 4.40.0" # tftest - } - google-beta = { - source = "hashicorp/google-beta" - version = ">= 4.40.0" # tftest - } - } -} From b31a22e610461c82806c16af392cecede0f13cc3 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 10:45:57 +0100 Subject: [PATCH 68/82] cloud function deploy readme --- .../deploy-cloud-function/README.md | 72 +++++++++++++++++++ .../deploy-cloud-function/main.tf | 2 +- .../deploy-cloud-function/outputs.tf | 46 ++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md new file mode 100644 index 0000000000..640432db8b --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -0,0 +1,72 @@ +# Network Dashboard Discovery via Cloud Function + +This simple Terraform setup allows deploying the [discovery tool for the Network Dashboard](../src/) to a Cloud Function, triggered by a schedule via PubSub. + +## Project and function-level configuration + +A single project is used both for deploying the function and to collect generated timeseries: writing timeseries to a separate project is not supported here for brevity, but is very simple to implement (basically change the value for `op_project` in the schedule payload queued in PubSub). The project is configured with the required APIs, and it can also optionally be created via the `project_create_config` variable. + +The function uses a dedicated service account which is created for this purpose. Roles to allow discovery can optionally be set at the top-level discovery scope (organization or folder) via the `grant_discovery_iam_roles` variable, those of course require the right set of permissions on the part of the identity running `terraform apply`. The alternative when IAM bindings cannot be managed on the top-level scope, is to assign `roles/compute.viewer` and `roles/cloudasset.viewer` to the function service account from a separate process, or manually in the console. + +A few configuration values for the function which are relevant to this example can also be configured in the `cloud_function_config` variable, particularly the `debug` attribute which turns on verbose logging to help in troubleshooting. + +## Discovery configuration + +Discovery configuration is done via the `discovery_config` variable, which mimicks the set of options available when running the discovery tool in cli mode. Pay particular care in defining the right top-level scope via the `discovery_root` attribute, as this is the root of the hierarchy used to discover Compute resources and it needs to include the individual folders and projects that needs to be monitored, which are defined via the `monitored_folders` and `monitored_projects` attributes. + +This is an example of a working configuration, where the discovery root is set at the org level, but resources used to compute timeseries need to be part of the hierarchy of two specific folders: + +```hcl +# cloud_function_config = { +# debug = true +# } +discovery_config = { + discovery_root = "organizations/436789450919" + monitored_folders = ["321477570496", "821058723541"] + monitored_projects = [] + custom_quota_file = "../src/custom-quotas.yaml" +} +grant_discovery_iam_roles = true +project_create_config = { + billing_account_id = "12345-ABCDEF-12345" + parent_id = "folders/321477570496" +} +project_id = "my-project" +``` + +## Manual triggering for troubleshooting + +If the function crashes or its behaviour is not as expected, you can turn on debugging via the `cloud_function_config.debug` variable attribute, then manually trigger the function from the console by specifying a payload with a single `data` attribute containing the base64-encoded arguments passed to the function by Cloud Scheduler. You can get the pre-computed payload from the `troubleshooting_payload` output: + +```bash +# copy and paste to the function's "Testing" tab in the console +tf output -raw troubleshooting_payload +``` + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [discovery_config](variables.tf#L38) | Discovery configuration. | object({…}) | ✓ | | +| [project_id](variables.tf#L84) | Project id where the Cloud Function will be deployed. | string | ✓ | | +| [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | +| [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | object({…}) | | {} | +| [grant_discovery_iam_roles](variables.tf#L56) | Optionally grant required IAM roles to Cloud Function service account. | bool | | false | +| [labels](variables.tf#L63) | Billing labels used for the Cloud Function, and the project if project_create is true. | map(string) | | {} | +| [name](variables.tf#L69) | Name used to create Cloud Function related resources. | string | | "net-dash" | +| [project_create_config](variables.tf#L75) | Optional configuration if project creation is required. | object({…}) | | null | +| [region](variables.tf#L89) | Compute region where the Cloud Function will be deployed. | string | | "europe-west1" | +| [schedule_config](variables.tf#L95) | Schedule timer configuration in crontab format. | string | | "0/30 * * * *" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [bucket](outputs.tf#L17) | Cloud Function deployment bucket resource. | | +| [cloud-function](outputs.tf#L22) | Cloud Function resource. | | +| [project_id](outputs.tf#L27) | Project id. | | +| [service_account](outputs.tf#L32) | Cloud Function service account. | | +| [troubleshooting_payload](outputs.tf#L40) | Cloud Function payload used for manual triggering. | ✓ | + + diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf index d79cd770e1..f30d7f2a2c 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf @@ -83,7 +83,7 @@ module "cloud-function" { } } -resource "google_cloud_scheduler_job" "job" { +resource "google_cloud_scheduler_job" "default" { project = var.project_id region = var.region name = var.name diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf index e69de29bb2..0c2c50abed 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/outputs.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "bucket" { + description = "Cloud Function deployment bucket resource." + value = module.cloud-function.bucket +} + +output "cloud-function" { + description = "Cloud Function resource." + value = module.cloud-function.function +} + +output "project_id" { + description = "Project id." + value = module.project.project_id +} + +output "service_account" { + description = "Cloud Function service account." + value = { + email = module.cloud-function.service_account_email + iam_email = module.cloud-function.service_account_iam_email + } +} + +output "troubleshooting_payload" { + description = "Cloud Function payload used for manual triggering." + sensitive = true + value = jsonencode({ + data = google_cloud_scheduler_job.default.pubsub_target.0.data + }) +} From 08d77f0fc0e03b587bb48d8cd3eeda5593c6b352 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 11:51:47 +0100 Subject: [PATCH 69/82] diagrams --- .../deploy-cloud-function/README.md | 6 ++++++ .../deploy-cloud-function/diagram-scope.png | Bin 0 -> 52705 bytes .../deploy-cloud-function/diagram.png | Bin 0 -> 55849 bytes 3 files changed, 6 insertions(+) create mode 100644 blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram-scope.png create mode 100644 blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram.png diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index 640432db8b..bd4d0141e4 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -2,6 +2,8 @@ This simple Terraform setup allows deploying the [discovery tool for the Network Dashboard](../src/) to a Cloud Function, triggered by a schedule via PubSub. +GCP resource diagram + ## Project and function-level configuration A single project is used both for deploying the function and to collect generated timeseries: writing timeseries to a separate project is not supported here for brevity, but is very simple to implement (basically change the value for `op_project` in the schedule payload queued in PubSub). The project is configured with the required APIs, and it can also optionally be created via the `project_create_config` variable. @@ -14,6 +16,10 @@ A few configuration values for the function which are relevant to this example c Discovery configuration is done via the `discovery_config` variable, which mimicks the set of options available when running the discovery tool in cli mode. Pay particular care in defining the right top-level scope via the `discovery_root` attribute, as this is the root of the hierarchy used to discover Compute resources and it needs to include the individual folders and projects that needs to be monitored, which are defined via the `monitored_folders` and `monitored_projects` attributes. +As an illustration of the interplay between root scope and monitored resources, in the following schematic diagram of a resource hierarchy, the root scope is set to the top-level red folder and it completely encloses every resource that needs to be monitored. The blue folder and project are set as monitored and define the actual perimeter used to discover resources. Setting the root scope to the blue folder would have resulted in the rightmost project being excluded. + +GCP resource diagram + This is an example of a working configuration, where the discovery root is set at the org level, but resources used to compute timeseries need to be part of the hierarchy of two specific folders: ```hcl diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram-scope.png b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram-scope.png new file mode 100644 index 0000000000000000000000000000000000000000..9ccef15892440f45df2c3f8f8ed226dd8edc731a GIT binary patch literal 52705 zcmeFZc|6qZ`!_u5DitLuL`)&dnh4n|OP0tk23M4Q-dVcrod0wyQkLUJJ&GI=v$9bH`@;=_j@tN?4TB@f{ zv7LfIAg9&tE9pWY$Ach{KP*lh2k+dxo}vN%q4CgFy$dPqxU>L)T!yGA-O=~`usqh{ zZaP#2CAl2EeE<0$>^kS!H`Sl|+{DWj`X6CfVUqf#!(^nS>4}%uMx5mpGg1r*y>GqHW8Qy6wv|h8$~lD*Gb5Ww_Eh zmE{v6Ve$uybx7FxQ!f**MJwm(M}+tI4j%|WAYDGxn|+A1F1aX!+AT&^Kxsom-CSK+ zcWFcA_Bffc2-o7(MQ@98&aOSt)Q-3MG^x)537Lv79l?1ID+2*;kj^k5EIk7Fyop;@ zE(e#-_+MHhn_F3BiW;Ow<*q$x)6{O#_q!P{4Nl-Rb=TvjHREAek%8&$$g;r^h5M)v zlKjz#rs*GWL19>Uaku60ECli$d5)r(RV|YO!)n+>7>me&m6gY?Pw=>{8&5=&WfZfj zrDrK?)AarGYcGG!zWIsuh}Ccp-j#wf7QKsbC>!Z^EYBFTSI!@cQOUg6XCi06Y4SU7 z^K-wwPJUnJri`|eO?Hmu)UR3NYWHo@E(D_Zj;dOcSyH?87g;}Mf&9*laoZ2uqdaOE z;_}S1E#Zm$+DdkC!8bg&D~4nE`+(?SV>Nq&3Q`dh(tYh%4>vD6QD>NGjHTPm=;9Gb zNF*=i_$D=@Rk%5ePEo(9 zOn^>N8i%e~UDbKXA`#VM+lx}bw9_co5h7_{F;cv<;V0``dM>&-`DohRD^RiC*~It! z=$RR%4;I>;yE3wQjs08B%gZauWro2R4|gPNw6gqCg){7d&{rUpIo04?e}V6iJ@I#d z>C_SkYcooMZ59!0GmXcLwjP56ot|wrbacXlkGi`pM{7I;nCGyAvQHfOIzBeEV2HRbl(7q87I zFXNye{2hUn1e4cF5FGrR!O~4PV#M#LpO&^7(on~t^TsR>z zU&Cm6E$jVT{v{Dq04>3Fwk1~%k)894fEH0e+^w$#;a~$wz4WY|L^5FjY#Ns`7PCG5 z^Zsb9->vn4?dp7!8cqgIx$uTebu`=#F1z2U5X1IjbIer{gO zGnlmf9VZZ+Vz#qhwif%&x~m(U&Iab;Te-IJn+&ZZ%g!xF{{c~icAl z4UgMye_3L3idONrdc)KYVcPOl7q{X2zJ-1EO%}EL@@LMR=_F}FAX359tLg25QWJ`5 zWy*3oRVxacJpFEF<$cQpPMQWPsfU&I^HjpdO%NGo08^GdMK1l!yxQ z5(Q^`-QcZR7NNM(W8FlR zp5Vmh69Y=u#KAf1<(+_pJibbaP(MR}$yx;N-YYdJz#u_pfYp`1wKQmW8ssb?+^*%* z7P#BcsgtXJcnQzpB`YBxAO5Tx4&ZG{det4BA1NwfBFibsc6xUYSjQdO60UyXbP%)LuTEfgwWDGKSAVK=wMQ*u)B4K4=;!bIRLja%omx3H) zMhK)qk&>ftq}T2e%+bCpJ(`+toAvTuIXA`L1wwhX?U7p>Y$cMf8~g$JeuElI>C>?y z{nL|^tG*vPu~vSM<(ndW<6drSh?)p;A{@&6Wz7SVEJ3P#l0tPPh#b6nShTjv^D0Fp zeX9~`ZyMW|G&np2v{RRMXdsiL)I@Z=t+gy#N;dBj0<97$Dk+mj%C(Ep9CYWcB#I<>QM%Z~Dun$_B(zWsIlF7$WOi*&}M6m=?*CW#TJ2mhuQ;l&X?H zrbS&(ox0xTtt|#imxD=P*tPT1Z=BPf>7)uaDcO;IVFi7bfq`O8;G}+XQNvb)LV38_@qdHH>v?1<=!d^Sk$|FV?nCKH?cH)JwLY`WSs9ml{=Wh@1Ie+e(EKe;0r7OAdQF%G?3Rz^!IH7-Q#Wal$0>R&<2-Ss^C@)7Sbj-Q~ zKhi8_^S`3@hE;)a61RdI!O}uPQmKZvQM|Gj_$w~!eE!D-UhO{=wzg>?=|)sv3v_fj zNVjr)U$LC{VP5v#7#f`Wcqmmi1P3RFR}>oEtP5mgXXn7NLcKT~Kpur2?$g^-e}NUj ztV;PLsOn{4^cPhD}pXLLhYrs?^%IlG|h4u2|g9J=f9t^FCG-CqD=5rSNdKdEFO|%(!n6 zTas3+hHZy26nVI*^5~6?LAT-ZS?W?ENEp@*83V9_{M*ANy?laa&T%L_>J8L926F1( zax9*qsY2ajg(C}vg(;Fje?0<60)!vJ@U2%N?0PvVU%q@Ow0e~^9E4}}cvtgbA@q4j z$T$_JZsfv^R(sE#d!m)!H#8N8H`zH24uI-R>DpeQkg+Q?FN%gP%+0QA_0m8{H^z02 z0AtNvIs1KIWGIs&zd6;9H*)nj6rAd{`#qt0R9UFL)|FEw6C}9#PHxN$c1)v&2D19B zI`a>peZ8hwOLV2V%P0Ml*sSh(+3Y4uf>7rlkdO&gs#)bkG{yd$HBT!yaM(PbKPBr4 z?Ijc!zvZWaToyRg{zIa`O` zFHptSI21}y?M-4TKNKMVz)0;4`fP7cZ*o+0v`Vqm8}mCL)V`vgcxSt&wpSF`YM$H~ZSzE036d_t$27LuHRC7N04Desx|ElcGl=0)i$SAyX(=)+ zFXMcoyg=6P5;*jlgw|P0Z|fZp?E0ryKZ95V4+B{=V7Evns2Sayq0gpW%mqTP(qYJ9 zE`qJFx;Wvp^-n_WJpv?$aa#o6k4u9`Am2j|b4bW`YXTRDd#5UpHNA|SH>4y`cTrhQ z+tf`*Qa63{a#|;vJi1J%?6z&*ub;Y@c1Izv-`>sF*HczlTdR>YXiSY?x2O`*6uV}X zh&^$#lF3r{wOBGkf7wl>+>UCl^kGO1F`aAYW6%IT01wss66VwGC}HJfm2c(jR=+O~ zjPo5ug?}pYsGT!PV3wXUuZC&ovW@o*HyfgBHI0NjmEE53{3FIgkZw-5DaYoXJc813 z5!d|^HjxQ{e(Pq3bJ|EnVU9lVh)2|ti1jiz_sf{VG*I-2^hvDJ2FN991_@4|U_YJ6HA-yN<@BjO`7MjexTn|74Qi zG^~Ohm-oSsQq$(EXH*41E<`p}_Ljj0F?^kwefC-Uj4o@ovyVXiiy`#QHYI+zmPLUL zkEr30_2IxfmP~y%1C7gRw)9)R-O?eV_vsC@-yAH(+!uAw8t zwW44Wt&-^{OR?81CNv139g)so^65q3uCP8><@B*QNN0& z+~D%j>|b$i*#6BJB%ij0QkKbs+>9v@CoogLdSj5YO#*IDgM_JQO@+GGs7k**`?d~E zD8dxyJsR-!+@=b5Q{Jn4`F#-q44gj8OJX`~2@WOtTUnTMb(q`!FadLyNzSlb@R2>^Bf zik$ByN>=*yEf-j$2!j!3a&_KUINXS4lteQ2e+giADb>9$>VZA`A`j1ICuGlwM{mrL|c9_QASEO!h*B#RIvX{ z_hQN5@Hh?tUyO;Mh`Aa?ZB##h(%|Ij;>lZNKMNq?ih}>SWK$WA z^+=;ROh>`eb%SFNND(z8-dygF&d-s4cm}~wz=$9+x@|w{_f3N2>V~gZ2bI28TW5zR z8k-ih@vf6XJj9i5l0XW3GR~kdfyn7J_iT%fIzNpWkpz@UqOQSiwRzQ=nMw-W1lh%!Fed5Kfd@==>=V`IbZq3}rYAWy67&>LCLCiCQo- z^ZOrxeMlu!ZTtid2W>Q(*gxB6Utm@H@LD5|y;bzKj(YC5{fD{z)1X5C{9$Le1@wYe+-&=(YHV=yCXqsC$YIfa?s>CcV%4B`fP zj3upkaNbtE57e73hiDraib!jGp(>EG_4WGswc_d<3gTlzBQgPmneEokeTzAqsH$pK zXt1Y%`XxZ%^>u^jn5rO7#~^>ukcN zjYkV4V0AwOfyb$K77`Zrd4L_9cc$+ad6pfTmYNjxavKSPrG@KLY$ga^>j~_|Wil)n?`F4 zfS@VWcF5U)+{@QaL;s4)0hM3}+!+vb03lz~)@oL=TmDgH1F+g9P6JShJh|`!dVTg>~QRHE}>D;CDW(J-N=yrw(el z=0oBUSSCYWX1AkCfk~uE4P&2)(3vv`u>K8`f_|4E=PPLl2cv}P=CZ778)`o1Kpv%_ zWq_r>6(c!B1|+9vhgHNUzAbWeKf#Rp{ntEoqKX0DWpeKbWAg8 zaDEAaMq66@FTxf zk_~i(dW2`;vu|uA*nr1Rh{`D_uy4bz{Kys9<5#o%^cvA^DaxhRQ+V9>1!!LM)#;>* zJb0_FBN+3^OeOn7(9gdBZR`Ysd$t+qt!?PbaKTiLLoP49OJJTZLEyoctc@3=pH-CA zaBn=To!dV$Z{tk!eE*#<$1%Vg%LZ<7;m+MurWT|X8=%z!!DoZ!RhI+kNoW9}|5DF7 z*J)7*n3kMu-7m-I4FU0DlFx}yjm*4mc+}HDoE?-V2sDz1MNN3*@k1G|;R0_Z6vxZE zSJn^e=*#_UI1BpoJE{SPA0DndBKCCxW_h@MjWx= z4);c-__y<6tLmCNR+bDboyRJ@Gv;PoI-Q`Mdpidk<=ZP9V&g=I>Lub*(|~V6(!>^d zW!q<)4Zr5qx$98v(vH_dhmz&$I*Y63>oOS%{A_#%)U~imtM_=vkX~}Qb0Kv@!{Z>z zfjCNLe<|+{V3aJp+Fy3R!3Ns2+~x@i5x%{(NjF&&zjKU?fUKw`En`4DX9oppGz<5) zE8#q9TrF3-r5XM2)V0jI69CN#(1D$jQdA5xh5mT=u)@icW+2d-J%Bsbwxd$XMu zfyv${-yfDt5YW#6MSPgASG^zW%(^rmz*=Uf(L*4Leh<-j!{c68`1t6?OYbLT>t`UW zPo6_>^10ol4{Wc~0cRupd|mrbV`{PhuWtUAj{f>KuDt!$$Fyz9Q(#GD*Zq_90@OSK zXp{fwQ2v*ff;GY4|8BGM|G`r4<9O7DuR?5AUCx6v963p5Vl|``B;YUPpUv0MHsDJX z->Ls^tAV=6|6(s!Jl^n7J#s%9sNI$EL-3IXDwhK;qU5TZIiR#KJK}o$Kn2vVl_xqB zo*lCB;Pd}K<+<}cpm+fG2`e2b+AlRJQjp)qI<40cJ+`*YgNszSExm4KyvDZ^Z+BL8 zu6y_sHs8JDS{V%|g$Ccp+t5OeCsNQA3KyJ8FA>|Ft{l2;glJ>!M z9RFm+9x9lH(8R)cux62Mb`nB=K7dj#fB#ByIe0MN@aMM3!ByNT$e)wcGVM=eGRbQA z!grJ(S?ZtfIy!zbqp41|7W?O$r6zVp5WEVAiuf;wt^DuD>?(Q-L(K2LF$xAm3Y0i9 ziV7MJKES)|2VWpVzROT+kMCcl@-f)7gLR`|;l;bbNfvcR77qB!miX1aiHW{^#)Hi| zmGi0{2Y4=r{VC}9K1D&}?+>-q^r&9@jS|?iIZWu0>%r90WX{7dV^~z;wJg)#%fGM_ z|N6EZrC<1`@rSbCrpz*Gtn|QN=C4y7oIbKdI>=p^@%1_Ia^tO!7x>~0K~HiQ0w}Lb zkLp;g%yngGhEI3E*5s%GDdh1-OlXj&A)paqFpj^W8Y{DO(7dX25F|b4Q1OC)C~R<3 zm;GB>jha~)*JTPHKYty+NR?@x^brbz)Bm`pW0Ay2e$D)k%U;GncPR9M|Dn4z++Dq= z07x%IZE8$AuDJB&YSkoBdI@_Cd#+wahA=QBEufEaq}(L;)0p>`VouH8i@~9x=ogup zpAGGGat$+03Y2TQ@1sheC3IpfJw3&nK}SYHnhk@o5(Tu&`-R+64a|`mV^f3$r)-RR zR+9Ji4<)g&v6X=s6Bw#0(fc|qJ!Pzh(!Ky4th_#Zeg3k~hWXv|y^+i~Z{q23T-bc4 zOfB8ZyP#l0SZHWu5dgV~22&Hbo=Qc~$!P}Na-E`qUg4J>5)u+@&*=Fcu zzy?@4dTx-sJV8TAOiWKyhd;~Aq_J*WlJQ`FqHeo5Ha5%zXSvk!6^fSwqZ<7gLpk^d z3n`!)hWit}H6O$SQRsSXC9DDhB*ENArfx2$MXy|*evhyaQ$*7=VSAo5XbU=LpgD61 zV`XKRu%+^{lrCLCHlUcLyNHVTTq`Fy_0MQdSsw&ZU{J!(?4J4*8jE@x2)?1BLEFCAdf$!_@l%+l7$=b#>~JXkHB%o*sYB8t7t8fS~9j(D(}jufL%K2sBQ zHp7gu8!q!qjG&0jFn4Jj-@k(<|Ga<2Iy-0j*DTVw(*1VIv?m!xxs&hOhTrg0*E^v7 zHQl|~OF}8YDF>bjx*rtj=K8mC*LnQ&4fB-?3?m0e@QF|kq2}cceL*4N#%0Ze(NPi7 zMfVXI6rs0rB%DD?5Vv@KrahJOiKn({bP+0rRDz8flmC<2K+E z8S7+y#k(G_EI7ae;Z2J|U{=M^*$F+@@BKwKkQ3O&ML=`(3P*Z8%wJN3N?gDv`xj|% zkthQPGoYwlwe?yxN+6ZB=5QQEYU_eAoU+Y!W7CPKc)rf*UIbpr&wbi5Qc8eiS7yHF z#@$EjAuJ8Aj8H!L9$OjHRe;XA~W1+tqXS-o`G66+FC3r+FIPSKuStVw#+a9=m#IXbJZ|aHWCnj z<@B)QDYA2*mkUesiYl?T@liN~&??ns7~fW?drV==s)(1pJ1WKeKnD$+_9;*~uKN{b z%5{NBpCR}r+b}SOgGo_dXGjKJT`6h*^&T-~SYjJCH9gOy*z#Eh?&`elX2bvuWwXZ4$go z#QE4r$OY`*1XLusKMO(k5*~$!Rt8J~{0hrU_~Wy9jd_`N>m^c^bAdS`Q*sotvlM+^ zvQP+Jz&?=ikQ6;o;lV??co%Tx(pjw?phK`B%L;IH{A3P(qW!w(1w2j{>v40!rPoXuB$!(AlIZ&!Xp7^pWKwBjZT4p`(Op zao;={>APPc@tNon~)<2nt1RUub`) zQgU$uPPB4##vP9IP_3~N_DYnuU%yfb6Hm?1E%LXGE0XZWh43h((986*nX0cG! z{fcq=qpr>-1%_}UrUr~Ub&IBfairAb@GOCZz0}%2k+5@OAmo7QtG=df-!-@p7+R9n zat$VJ2Q5Q6o6$bbCovGNjx#|W#hH|_0;o#YJX47To?_WIky)|u%FR{F3tZFgS6CA0 zke{PUF**eoN7@Gzpuki1ciM?=#E$mLAg={@2>PMGjGF2+(N&17gbP2jBbbh4u&1>S9FX52S=U&B&?o<3^`r@j;b7 z{2SN71v${MC{xM8Q-Qkg0Nvu}CcnShvatD9QlerycyBh@*1v@Aby6LCZ!fniN1qMR zseoh}c*p#g{B?ozr`*656Gms{Ha&f-o+-Y7qJwI`hQ>&?~U0+n)N}l-CpshmT&k`MNe2E?j#_q}h0&^9QCc zhp@5Ty1v`|914c=#M!`5RxN&zlPf%IsWsBg3uG&I^ZUCkSwjn?jm%`7Yn~FuWMBLp zN2fIN47}{gpjY2P&>0OLM6%QaW?dfXf%(mFWe;@yEBykue=s-g%@>0buTwv?>b*dDuaLbwuxMM3c?iyKHY!N&Fs zt)U)- z+L#p}(3mn)XI?5T?MCy>XX4@N(JoxHr1oevo8D}A1^oD4hViA_eV*O6 z5||>3V3ue){|D>N0vO;6IkqXjRtHNrE$^n)7gtLK+joKxR0Shz;vZR&oQg?9M#Ab|e=JivXz5UkBm2>qvS?7o-=hQ@j8B05* zoX+&0P5G-iuG2erGKZqTvIDy=pnWj^wHE2MI2S(Kw4ErdT{|l_FA2zqmGLTOBon>} zjE0hC>u=+eiIUIL=mU`DjG>h7A6%FPI#%VpzCh!NwZfY2OOE~$wr@1@K5&+fUv;mg0=hP=sOU-tMWOoAv<#Ibhi5$)~dkB10PvucHysUXHqB_7zSKPrGV? zR`b&Q)^!k?XBvGYE)YIi=j2$z9XEVo`CzihiZTuxwdmN{v&IkX<*oF>4%eN8>^ z_^{;_=u0NFD}}4pg1sgGNs&f7RSwT8Vvcm24X3Fnm6~v)PpdOt8cdy)1uC&GSi0Nn zKES~8B6nA+umJVvF;`Blniyf27MRqr^D8b{LN8sr@m4q;gPk32CZLgFmG%~C&w$Mq zGRq_^E(O(XJe~m2>ntOsd+rcG^j)^zcVrb}#`G+25Gwne(L2rk$RO={ZIu*wW(uet zy#4u`?C0YF;Rd=2XiUxC(bgK84PzoZPS;LW{py(fqZKpRNV->`+$(q5{@R8YS|Mvw#}z?GB`hp#v(V#hYl8JyTN>lK8I8HNOSeabOZ0Rt%5NXO$I74)@N6gCU}U>>P8E58XX2qdw6UN&#LWp z>==D{ZlMJUG?!fbcp>=AU1Huqx8s7m+nV$#`Vx&>%J??E1Pq4 zbU_fVjCHTlhdO;LsOSJZtyuDH!41W^#!@ zAtWKtY!e*0Mp|AXN%!i9&>mmxZpI4c^lLwC_3x3EiF_+0&l+3M&8Od%LJadjZRwxu zpAqN&(B>)LNN}hc@^J4Q3R|x!6d<*v$GKFytDW1f zZuoGEFIY{Ve?=cCI)=`Wo0~FMvrzdg!zh3WLLcF=OtLFC>6UVPRy#iz-gbRX8j(9m zo9?4E)~3Kk0@F-4w9yRsnI{f47H5a_1#4^14mnq3g83ff(~9LE91DojNe}oC`B~#3 zA3b6?6via87Dz3rJX~vhGByEa8Q!liCMdX4dOK$ba@0 zKYY2Y8}y+!?v^ZN-0gVv`@QEk_Ns2zD<@fR+IW%SlS}5rCEIgv(ggWGNPXG#N0_UE zL8%;bRkMNi)D?%SO%GY9*DcVKfxK$EjVBz%Q%K#vYy2z!UYVOLK3KhH@=soXQtRKV_i$Y%gJ6Z69^FMPCFDFBTz{KnS?V4+KL#Q}b zwE?p5?}J1!0jyu%!UyRb?X;ENF>gr~W22w+?95+i9$%qXdG}gzJ|pta`X4e+czFIe zckbqoBhMJ`3mRXEeA<XgRbpwejAe!!K2A4)rtdOH^Clr93XN+9+Sn+}{@%{ccVa z{v=ID5{Y}`Mqq0XX1`FjA^ozML5(72Hg)TYIbYHL`E?iRN;u^p^r-~R{CPG79tuU0 zbV!k;0E*-sg;dF<5;0xZJ4_aZoFYP)TdX^SVBj_h)BkasPXut=Lt016%@CuydN2P2 z3t69kKB#cL$Se9A=%XavSJRXJa0SX z3RT*JD{R?pDCAPtb-_&)U5apa{}S%sUyBFb=Lc(wQ>Rk?%Qe-Kxzb-!wlCg6A(7Jc zK)2!(g%awe(suvzYg_Vbil%I-bEgo`Q_;{j6iq##ju?hew#`RT4|NHteBciuS|17b z{*#n{DDY6G|B@&K9|V#z`K5~wXxZT(8#xKSLgD59{z|&J!me9ALjmB)^m1VnQjql1 zhvG4P7$8x%AWqT32QY>JQRD+B+CtsZp>qG&`nT8Mg#Wp?A|{jv(1(BTm$*ILno#C)|9&f-aPL0}(c$nv3;OSZOosBv2#|91!CWt->-eFWoa`{AgdYLBe{32Y z>k}*}Q}_0dKve(!+P~JfIi|p90*v+Kp@}e2zWWaoI1=!0uYpMjCd*L{3<5jkprxMy zIeP!05Doyc zSEKnyA9&y(QWpvL95J(-@wO7Y!x#9@VcC&K_mWpeOF9qnMPpt?|ZY6JhaX(Y<$4G18ubp!&VVerY}u@KvvN&qQST zN>s3R&M{m0epOI;6`E0zgHq<6$vAT+qPVn*bBWzl_3^mc*q9j@+%EKc0>S0xKMX6p zd*w_-)2m${@SK{Di$ZFraC6tg;HYSNRii5t89MGdU=*_WTNUF{vZ)I>ss355!zCTs z!!ty>2i6oZc$b&^0E{Q;j&zKoKixdP`sa*$HhrA4hXbD>7`p)IFWN$E5FdD6IzChC z`>ov>3}SjI6k+w)Zno@kJ(!#P;^YWepUCdmkE|EjejibP4O*8DBExb4SixRNd5Lq- z+TepEmE&C(gGh7}(M3@C@T}QBJ%oWnnen5su=u{{2Ij8SKj|6lXs)W2nMC&i?(We@ z*mR3`r|5#GbT)7E^a~>13^OksLUm0fn3|E;|1A36+e$( zite6z#(MS=k2(?-4is(t$P}SI+g=pzD8W>blQX@y>r?3FsrUxOpu$HI$>A=+OLyR; z$)Gk7Ve}~35-_kSJ5`psmrB+(L@T~ty7yvFR9co*uA(%=WH#M6BEE0acw{ijezV=E z)!XNSrN2E_HZPhxt#^aQwj3OV{5Hu{XD9uV$9g2V{5&mflb(go?;}YT5mO(Zv9e!` zx@%7N*}&VIXzAr72A*8V2vQ++jd%!HXY*RQTZ-SrJ~-mxM-6gVlMUi#Gehv|7?@!S z8$vcETEP5q@(QLSGN3kM0yG!Bt`S00V^=E*55Cg z39O2u#Qq~y%pwOkqkhRwN8%^lTvm>YG4FE;tekAs&5S+paV42(;_qaEwAQ9Q6d2tc z8!4_TX9OEB{#G8ec)P$bzqtAvbE|-Zs!%#GyZa-J`@anG`x@RYFFI_k+~$o_LpK68 zUHgGjaN6G0!ry-NpjjZS%X?Ag?jp&p{Jl1E5^Lh+&|=7MY7n^Y@!E$ zzGm*aAmvDOY;5lDU?+Z$44cQ{!r!-?#HB8IHfhv6)s{t!E05@H4;JYNk2HEruGc24 ze-nsR&$6_16z6SL>2##&%PcTmmPJr!Kg={`PP%$c<9pH5z zY9{KeSK)F%eN}Eg`=Kz?gZ&rQo#1X-@#~_|AFdu83N#v7y6tj9*;!g;AiOp8~5L6MH-nCSmXsn`!LtM$>e0$=bt%BiNkIuU=c`wj)h@;~q*ZR4Dyr zlkuZKJvv*tcSil0SVNyEdWL4OLm9uJl}F}g3Lbb)Tq-kaI%|B5FDCbC!LM>XM6)7( z#$#VgJji6llg{>fzodkA;8<%-2y=pYdC2jMx*{Xe>KWm&mG`M(E5f)`Z;!Q%(vl#S z%J=GgslEB5T;Cnt*rkJ}4e|%TgDTr+N33IV8Xm}X^XkNyC}Nk7c|H`pwR}wcrZ{Vj zoQWcf*F(XkwMG9g4#d^F@@X=sfQ|I;8z+o*yqX$39PP@p3Mo1M2b0$^2Vty#W&fQ?5nBcbH-T7N7 z3SmzyEr~UtvsEgd5Xvo@$R`jH=t!AvW)ml{g{}Xias7It?{U2 zp=gPcPWGqsWC z59mI-S0DSmZYgqED1G31f&lPC#{8vWIdSRRXdl zS5%by98++5;^R^`W&bO#QxaEDS0oAC31iBzg8nh>-e&&kn0ObIq66GiTZ7qR%&PJ>WfY6{2})ru&a%h@^2WdYI;rQ;(~ zO~;CNEw{JFYF>b`)z~fhVF^HxvO5!rM6;pJ#pO)rWVVZ(j+Heakm@mI3bCDTFZZ(i zVejetS(*D|{LS~8;PLL}Hf0k<_T=Jnw`!mQowysraAsm*1Z)jBHV-8rX`}d%g&zrGH)YRL z!eWJFe%ZX5PY2QlWvWmv?hXY1Xq_ra7vvmO;Y1S+98EE_EF@-F&Vl`O>*=(&fX=qJ z)I^1Xz$1Ms$o%-L7e|d)&bS8Jm-#O**ADAdVGW7#!lFVB_pfgKJqrNN89e#|4is_Z zIfaz@T^d_}Gh#A)pwK<1;=jw>rMR&*YF{5#gno>&0j-OQ(tygh&-QD7T8pVV;GJFY zsp}T5dcT)8Utz3#jIKC)?fyBoAUtuzBx44_vw`D67{Qvm;U1uzN4s&j z6tjBmG7JdwMOo!o+=(l>v1(LXXD3u__7GRK5j4~~o=UQFkQht=0V`)TZx z%A!>sGB?L!<>m+a&mDF&edeVBhQd877dc`U)Nq{UE>C=~F((*uO|PmqMYlWgjX3@3 z6)%9oi}?6fLE-dR5N!rc-|AJ(Cp?=M(kb;@LZ2nInZ9mGdtQgvfu@@!&p zm|;1|c0Y}pTT?`-gjQt>m(Ffo-jP24<$=X+r}s(hE>n#+UAVa)q<%d|9<@cADuVMB z9eE~{u8TaEKP*?BS^un=j&VS;+|u4%@SD>Oo8ARO!vS_@x*%TxkX;C*mi5myQpFZ9 z^YGZIXFs3H%}NfHAe&x|ROIlPh@X42@|14AftF|mQl2#nvy~0j)8S+BvEM{Xk@Vz& zBf|>Untu&fc{4olu(4jeMhXv0G&7EPw!cME9L}}h^luZX_1Ds46vYI-m6RF>V8l2A zxeb!&*47rrj=eOb+-J+u%M6vn4ZDZo;{t=OH{}o3lEb8`dk)-k*`PkRiB&BTW&VQi zERKl`Ir8#0Cslu{p3D)VU;-$6IDthd9k~tgiE&KNMWJ*@3HHo}qoOxuJuxPV;2BJb z3Li@YR4A`nmaeGWZMWY$l*ib=mY%D~fqNrcC{So=(4MQJ3T3q#k?^+ek*c0BWmX*b zJ~#7)Qh~SBkeiS7w=g#6OQUG*M90&(()t3OUSU_4wnh{JglERdKW)dP0JQO0$FU32{tTZuuzG zOYUjjpICP{e23;J%AO)2kI*smuUXHP?zmRT}pSxHX zRKeux^N-gBS5UO#KXimYK>)19>anh&8aIsQpx}vfLq~?eRl&x zx}!~hJ-Evcog~IPpjM6%N`LXfvaLg;nf#+C+JVNbd`wL27NZsLqL(mXcH&AKc!;AD zc(f0`yLX5*UoTguE!nk9*1dGzLMRh%#uGXkwm z_{@;@p#VHnAc;s*-W=?E&~Wd?R?=jFS?MuQ%$@5!Qe}(cOHpt73_Pi4q?d^alc}mv z+qbky+99lyNDaHOWuA-CI{fjNWOOGx8=F{foQ|t-qpAP)9rD*>r57pK7Y^_}YROe3 zow+QeG(gJzL3CPJ7@;_JKG0tP*g6+i2CN8;rHv;!8%gx=7C1-Y?1gHtOB@%*kJT_O z1V`D9y+TNjxgp*_{>VnTAQahS=iOv~_ie zeCA~)a$bbSf|WwwgZ&idmFAvmbR{!n+tORdMaVhcSvS& zA!}Z_u%gtLIo8=*t^2iz2xCjDdL2Sz>48|q!titxjdgd;m2P}i!Zi5oj zMCzunJwPR#3ds(YF4q3j!@_ggecI+pDkIn5sS^(NrV#A?m)f6}5cc;*+*;jlg&XAR zW2WM;U>Ygd(}JHXXli$&`^ltwO5YLbLcspduRe^Tzg2O}^Y;oUW^1hK4*0SxmdC^j z#u_e+fVmh2GA2y|);`f^5JpSH^H>j^)DmFUAhgF}b!5E>O1yF57CTeKWn4qNbpA2V zQJ^0AkDZ}vr zEvr-UOr75zd^vYC%OjC zvAmlu5vjAQ6iIGxe6i~TQ;JojJft+=vaYuqvNsuruzKSARXH=G_Y$@6If#0yF1FxF zf4U}sZvN|72_twM&88v%#7}EVL7=xcZ~z+J`@_l5gPo-?T85;3t2)3eq{_QgPnW&c zYC^wsJTGp`WP^l7t7h5XYq*l4g%;VHCT3vyUrcw(N0K^x#>u3*T_47}{Usyfy4u@J z=C0`;o!xz}Z$p|pk*j75N#YAsQxNo9ugr5Z1NLphPEDzELU4tp!LEiTPr6kb=%}rXk0eZ89lC)sP z_yNt)7+{QzgRbY-j-)Sbt=-%G(X;5l*9Wckkp$LdSN}6zr^G~YSZ-7wglodK5rg(v znii zBKmL#Z5mCGNtvpQ#3JX0DHl$;XueJ@3#HfxS<~L~OquUXA)k+J*>5Ha7+ICM?^Mes zv^Ep#c5zFOuFp(0l(5?z%uRsj@ah6KF_(*ldylYPWQ*#as{e^)FDg_RU%9;`=Jc00 zLyK;HAF%h@@TS>#Jp}OA#8g&?ot{uw8{j1=wAEI}crKdaAq?`{QdOp3b;Blej^c1Q zG??ciM?)o0r?vqv7QSCwd+oe?Y+dsylbmZt{2+~41=xsya*Az>a9cR8rI8OB%nIQ6 zfb3nViT*9EHX*^LAJc2^rydwxfr3maDn7_EGJ*;XAbt=Xy1*zZD#6yP2Y@k8Pcs-a z+vpWx+v<>6p+M;QrSS1o({6)nC!QB70vhgx<#kYg0VWaq`+|YU?D{sTo^K+ic0ARA z^U4SOQ##v)#+zsP#uis10EalgguJkn47jA)?yo+5YVS3rc_|NH*>m~tt$6eV?`Sgy zV_d4Y31|fO${4EWilCqnx>Mm`CmKP_(i46AK-Gem!z@?-NxQ|Zu{(0p>4u1C^;;78 zWm3KkUmj|A7Jz$0>w0g#O<0?08LG$B?GG>p;=|{b`3C9ID4rYeLQQYktbnR ztRJ%nhHXd3!wYaJlAL<4LXS_!Ar$O~K7>r>+%w28^Icosc1ZZ*mvUDgdYj$2uEeB( znYPLDw>`FQnYURLQy)Mq#PhI^_p~y1ts`F!j>!djH7I*g3CEk+d}t1r`_;O+mnLL` z*w*PZut83scD#GV1vkt4S)X4=FuoJUzmz2-rbigB8F@F!O;De8Zxa!0S{}6oC<$t5 zu4G~1(x&;Kv5^F@j>{JGz^ZYAoD3p=OI#SrrwGnU`AwP41~ed!gH{HtxiSReVoGx( zeGDVQrjN--TK6ne2IIFji=+u;zs3|9aG}M1Ygq6qL!pc8=!sq0CJIppB8T2Ck3}=U z36<}pB)|9fd&3EvnI5JNjEqB$5^LMAPq&_pKgxy4mpGAh!R!yiIEq~_4l{5gPm+o3 zSqWeud~d!5X~gCp2(aaJCc{MCmKXHnMd9Hi(psp{WJenx$y(B)4wx0K@V{1l1Ju<} z`ATSo$-}HrAAo|pG*oe^3*qy)?v<${`whWiq%w*fcQKy=~t$&q^T8y2I#oldKR(09kw90g`v(NL-!k&*cl`{luZ>yYV3)c z)!udn_p1Q!Oq%PC6C0U?rGf5(k9GDmAVXtYO84&qf}hci6(UsdrFrdFQcFH_W#8BZ ztIW{k;*)S&+CKYRpg6VMIvqeWgbMX80Z%Qqij1rJjDOs29$8qzfgg>F1qBu(sf+mx z@rX$TQ0Rbg=jg-$doMu4(G%M#VRd^IAOfvp8N~+bD(1n!EC6-QJ-P4)W)(43}ZCO~&Z4+8BL zff;4+Q(l4vUz&dXGI{$8E2(Rn3ij7*p8L=6pY1 z^!E0)@(vPxXJ}~UYkMP`uPngUWp|erqyoO*fJ zvdC96r&ksK5f~Eqy`d;Xj8gr#&!xqi(e1$WBFdtpS)m-@*vH7b!BE0d-ToZf#-`I5 z273YKl0jMv1KtDV<)?6PeRM(zW2a}Q>>h~uC`8dVC_R=}R+?rS6TKOnT&z3<`X$RVVAoZ%bfXs| zJ2SaLeB#uVpY*(MAa*(OetYk@u`x=a0hLMv%=xb1S9sLGqv-D!S_~O5KS241i7eve z{J+?H@1Ur%?_U&k)Dd(T6hTR?ppr2Fl7osgf`A~Bp%qY)B+03p(Lu?G2q*|B2#5#_ zl0$=mBuLJwk(|LMbT`lqcO6i_^LzKcS9M?AdUbErD~y z&Hdj3;4SGUWj9w97<|8-0NVW&#WOc%uA%-RGb^pPeh+Ga?P=bk<%`1EvA(r=@L<*p zd9bT7+2iZGjPL*W?FcT<>$k^33Ts*!Xu65L3Kg5{dhAxlH&hn0Hs@~aJ}lM+t)`Au zaDeewa(B-_?{bkuMyuH~I>tzDr%H6icUD8WodLB8XVq8MD{z#(Pwv_Y4;Nwuhb3^i z9XvavUm#DSlkTZl(_?}}ph7QZN4Jr^_CX4i2fE>OAUp#pz|Plg3md$V){W%i`GtYE zIfEsKi9?T=K7Y={l7=3Cd*pW0TK?D$7%@7!8d|YJSIL)6w@m{}+6$ex9Zj;$ptb0Q zN+&9GNXtGrRV)g2j_tjjb;56}-F_Ax%Y4I-of0re8we;auGj3BFkh@Nso2{P zsZp_d_9NqCx~-{5_hh6HJM*0pkxZx*iH1QfSNIbHrP98wvMP`u2Q{M}J8pm3FBpUu z|1%f-3L!W%lKbuD;olS#KixKhymlxdk*=MtjJID5u~1EG+e-4WqjCyI32J5P0-NH0 zHQRuUVIWza6)Bk@aO^;ee90|BABcrg2vw=y|8%O`4+Njy50sAGUj!Lv_Pvtr~zdRD7+8Fz^YA7+w~ zDb2~v^`7s#n9ao30tBd(*N|^>yvtzVlW&HYsc?UJxXcN>$A|lDVQ(PYOiY zeD*Cnxz(MC485QXumo8mL<`fe951Hp*Z|g_e+aicqG=??MmQ1EQ2ljTX+L9n#3kO| z?tZ|^7{DJkl4I8B?vt@9!v9ltFs?6a<7zAvB6YV+P0G$LM67RA)7bDM1S;jNO5u!i2`-cS5J6ZLXzYZ5}qoUMt27!k-& z)%rC%|6g5*QjMp9{Ll&(HFCQTmqB>*KeI+5H5XacF?K7+&!s=2s23{1;xQ?#_YC=H zNpnW1#&42?u>-C&2|c+4V?CwUe{43WrUWQZKL=brWw0rquA_#FqO%hJX(#*yNlS@n z>Z-!vwAKlXJc`)2DL+h#|J*k-!?fA7$q81>w#HRkAnNOxx#Vze>~eCzASG0HlVL}m z)PL@gSHMEl+fdCwUl8q>d4zj%KAT*zNgmo9Ut+2t*=BMwC>p^Aw9hgeGK&2KmlZZG zVW%qaVs?2_u~Nv5U4S`9?{^+#QU{TZn9k9^eC=%q&&xx!QQl0h9g6J8kGD*m-}&6q zzG)`9Jj5#g7en{6OM0@Ybwku^cr0ZimBvXY9rn{ADKgMLqF^UlNPT`>JClVtX|0s* zL&2lbJn($(_dVmjo3DSn(?D5w*tlbx=qFygUZwN#|0fNv64<1PJMJMSCDd*?}8D zeO*f^dibS&$nXVzFa;TpRt!__6%UcXWPn*Hv=Xzdzat7UGvXVAkObn-mchZU>05Zv zb#C`^^Bv&e&FsuY&r1}VS-NL;d@g>`o>8Hj(a}vH1X?KTWV>gS%Iw**X6E5|Od0)< zC-RD^)$TGpRCCrhFbNKA$m|AFXR|V{{ymL5$ut~Z3fd!0o{5Cbpyrz*X8#|{2yVGd zDsbEdo5J>Z`JG4b8H|F@LGZ~lx$2D6@G73uOibyYzkG?aaKI^$NMv*DEq(~+%5aT;i4*Q}3Jf1R~fd`l2lKnOng)ZzKQ}o)Uy`tc9;`AfAv(JLYe>gyCN{;F zC`8ty$;~>W-eOm?w58(%RV=wVS8G|g>E(7YF=-J7Luo7iZ+8U>nAP)eKkw~+8ZYPu&KN1anUvQ$Bj0E`MKiAs|Ggh=76_*pjCQ4rA= z)+uFZhD?H6Y>uW8A85*lZIlx-P%n|Sxd8RZ=p>rc5`B*b*Jm6cr<{8}$rT%tRX)TL z8Wa#N?}5GCnOBn_nBw1!**vlF7G%4D3wLaYP(sWt&si}WX>OF% zlEYraIywTWB5ONjm8B^i9|rMxk*kUAYd61p#Rr4y1+_$Cnl^{eJKlRRgutdIMh-1j zS$gj7dPsb_R(9y3bZgzT*WHC#*gCtC?hUJc0opsv6V+gG?(A84oBJ;Vx1@gWdryI^ z%7I*&uv0UkL0O>V==z=?G-k%52$iNg6R_aYhsv`yT8hMFkG34IKuM?yaN&W0n=K~hTKRwcovnms*%M)6e<*|896V8A=&rJ$2 z0|`QS4N@8TwQpEipL6UmXi6N7Lo|wIA^_gs*INc$FE=3`V^JYJmL*1SxUH$&hNN#F zLVI8a4ntYT=c4n2vR-VPU5Tp^S29kmDgfj!DalvhbwpTHqQYf^H9sOWeU3tU1TO!P z6JLQpkU2&1hzF3YyLUdMS&$o+!UWM0w;CgT=lzBNENBXDvk8UHe31PZM|COaxKyNz zD#M+gCnVUfoH=;g)lRN_ZQSXzZ=rH(QU)&XGS6^Q$r_p`vZ01F9GSd6&JvweSjcOy zsafD?@j*>tqsLN9sJyo=4G0=EDXOhoddm6UV@q9|Vg)>l{w z9xRQW(1e|$-^k=)d3m|#25Sq@_U8@d{w{F5CgkQa*%sJY=H?N)a-f1W9;TI>PP1zi zQf*v%c}mGv-dzm5MB>y0MxtMDlKtRX|FL2t+M|Yd=FsSzrr#gaAyY}xvfknx@gq&< zVj?@(;i?A^k;VDChOPs;3-H&}4cdoX6f2cZn56qrv*TmNNV~L!R8)rUQty;xM{L(0 zX`C48`;E=~^-oO17n0*^Y|~nRlZLw1VfL1#_LrM#uQH%3`zG0r1YcapxV?X7#R~z| zj&AOD?0}L~U^m@eWTK?^GlBhPKv&(qo!di!+3+xx)=wSo_|lx${A?#yzKX_{aO?__znWRd&42v_skq_hVR(6Oj| zbFv2tPY;TZmcg#R>08YfjESY_Pis2RAu7!|hM#-W@y7!M zjqNVaWe39~%yrZy)qzXF)??u)kYGTT2y-ww0B0`<%$?bMm8l09*K#k4WH~uLJa+v7 zvXkI%e0RS0S7<_lQACcXXI$<(RcEmc8K`#3UcG1dd&JKioC*`@+;4I}vUHa8i?@dSrHmtmfO%0iI zP);7k$0a&a`0>bv9TJ$$z=_;+BthbbL-DsXUv${{v@pqmKZnL;dcJ{-z^+B*<Z%2CI?{cPrN#&0_4qlx@wY|8 zvgi2XJjGXf+D9^e`JES|G_Go=fYO53)ok$}9TEJH;9qxSuptef z|NGJtyEoWIyKmF&;T7M?D7`J|g$-Vj36?+m2)yq9Sv27PykDrMH2#5uwHo|0VuWL4 zPHZO~(0w}x7w-T67yb|E676&e1&9O%q!ym zj`L^)By*+})4^wd40)Cxpk(%zH5nH3TI;NcFaabRy)wHkk>9h4!;5+6}Vk1To zba4>F6qU73-*WjI?7qIKNqB7JUj^6Fbs&H^39f7=7rlf^v34NH)A*uQ;gkCNg7Tvo zw(OZ)%4mp5!}CKMbQvF<;%8AiD^tN7^)|3@WO3DJgfo9}rs6Ss1{RV>8L;1aIdJ$J zZ#%Bn-*;My3KK7m-EK^@+@_j_j6txmNnZfS&?ct^Dj|6Va#-|vS01M8yhR8kc$QYy zi!6yW(S)WulB*?r(RD;0`?H9Wz{BboWKfLgy6Xii*Yx#59S~>@M9A_h&&-c!M@l(e zH8C^@PKnB&oGU)e1Bymj$>yX8VnI-q_B$xVF}-$2CnHVaTS}Ys{szazXC+Wm8oc$I zrQ#Gwg9&Jf_{jI-#R*;_67IVCib52oA?LQ}(G$2>TpfNlsuiSqFm_MbaAZw?-w#nz zJ#S?t`PQxylvPviefBR{HICfc#6A)PD9?|eLLq3qPDXJstD6m;LAt$QV{P9|%y!O> z>ByU2Js{~H?D9?9z|X9d(5*a_IxXaDZDU4dLszMvm69j*j?6dCn?ica^UCrIjj{h= zr*z%jA>hZrZ@Uz64p9G8sYTIpYGx+%8Ai4QUz~5pl@WUP?wnZv{LGBq zJR<^ng9qEx{7sSC%2VTYD}(@*(E>eZd5YQB@Nj_7D$>>mxQ_VOv5c}rOI!LI3%?N z1b4P%Y~}-H+16GA_t?Dfg}B+CjL49t0@K$ zh`{kH{X?N%a;n}&T9dA~bYHOPVZc!BBrSUK~8 zBNO|0tK+Dv)G|*EwY@uqm%+VPQo=ZB`b;D<4kPk=D#-5}PQ#f%Q@8h$Fjvi*K#zlpu7iCN<7yT$vrZaLb1Q&a{FB69$VWn?FH|R&- z_aIt7j|IY84jktfNQ^-_g}9r9w+EQj zQo7~b2n5KcDxvou-}o~Wb?hTSnj*Efr)M_s%4~|hp@E!uph*TkwsL+Rv?i~h56%Vp zc8K)Oe}Pw*l4mJQQ`g6&ef(G%2d z@*8Il8-$&9ZX_tAk=I$$*^Xp-Z5Je-rWwSwA)9V6FxFu z%dXwnz`{s#FspSni9|!`>&nMw{HT{HDID$&gK&P>A1pb%X?M(SS+7=}b$4MZ%}tiu zv+kXomR5h({R_eJBODYad8%4S9}@+o@WDU;yXzjdbNT{32t`zPm_&f^q&_1%IAwtl zIsQ_=7kHh|K`CjI^V1>siaAP4_tv=IwiG0t*Gn!o;TEWW>n)R0x^Zf?Z@J#{K#}34 zUArFngT7VodM=TOH{uTqIB`{@p+oM7n9EUrSAW#Yzf6k0JGaMOo)y(rgwii&w$i5R zMwR9J7vkn~ZpN2K?rtwk^JBhtI?;vE+B5O;Y?t?rsMMgn!pGOwU;7<&c|30r>R;8? zr*(xa7B;fG-8s!~%`3SOP0x1)==!r2#|E5_+7BEFPDCll_~47h4$09gJTJ(XyzZXB z#?BdLmh-IF<68+ECub0ElPSmiKuLKHvir0QbVMxHQm!EooX_h9qnvbp`w+~<_1j2` zPa0CBsOTA%oGz?Er998><@$#t8foX+O_p~{Lmu=hJ60C2kejWhS#n~c&F?MdF*j?2 zSozA1lZrBLXV+gcH|uDzS1|xAxi|CAn8vvIqk$V6rd6$E7iTjeQw@z85=jFv2KPd9 zTTl63uf5l~j-t#QY=zw<^!~uG#>EbAnC#o7aw%Ouq_ohXIy^jqqnte4;o`GM9N=Tg z)V!^3hpWhvvr($dj2mg`_u1DqN3Wo`tMuuoCM~?2NlRbbVMYOWbDKM)cD$b{_5CYf z2M724@FJ`XovSIK!(-kRW>bz=L&BAeGL{bNr9pA|n>fsV61i%A-n1+>Ht@^TQT~^| zS)-#cy?mzST~i65Zh4cSD`aOENYtuo4HjUgC$mLfWwWB=72Qs9 zGxH^R4tTrUgS=v@)uHJx_h&~?fiaQSbBgmhnV1ml6bi{xR4$x%PQX74$;5Cy2?(&V zw+}Ot;1!t&=BLsfn61kaQcE%OrML0|F+AaY0KNhhtR|Slx7+0{+4LVS_K5i}XY80G zmrC@dD~dR5ZD)9U={P&O`Z!0DdCtm;y*%(|x?oJ)$ow3;(*!A0U*A|-ztU7z1~PbX z&2@F$Zg}Uym_fWBv9-*YR9_#Q$<#TXA8;w^a@%*A@0E zM&5x-rp(*~ncd4aZ;$xO?X5s(FL9IB^OC*rH#0eJlv^cJhTPoT&F5pxrixqT%z3oR zL!t#L(-;+k5C;r&Edfx;gU$B3X0r3L2D93Rf`i5SdcJ<5FH1gNq9WJG$Y{7L5jUK0 zfK5@)672&(pU)Ca^U;EOQzIJe_p86knbFmrLc-bA8RXosnOtlTEMdhANnntytyPHrTL0xaFg1a3 zBRunm+1p2#`){$4@>_;p6`einRgBRe+G}<5fU|#3+rVMDfxY}Mr=ItCh>Nqc$Hno= z+arghX*5=jw%-NHrt&66Gq12P<;-yX?T@ibt;)pLc)Tj2vi$VKjKAyBaqc~9>t^44 zwG|_)CTAKP=v8C{8fPx`@r_IEUdB!(AbKrbA-vhxJt^!`6)%yD@jcWr{ZV`no6nXJ z8J=W5c_+_BI#K)yK63>|MYoF}%sspW_<9%zp&oMkD!h@jaWm|#sqc9IsxhQqXK5jC zl8#WSq;wdkDW!&lCc**W=KZKVXc#f$oxY_V%?eY1b6pnl?pFx+V z%5IfqGQY^{^o$JAjZuk;pux?d((WE#=NI;0@c{$*DOPjrLF1i`%xIO{?t%p|mger} z`SUdGoFf^*26lyV>~IazK4N{vbbCi_U=#5ec(3W4M`tDcEY?`Iin zOt7+_2{DtBA)^g+GiR#^As zPz&or!GioN{rQ6eGsG&lbsQZqiEn9t8b3%gSn-;>>*^~P9n+Px7(EjW=CUf@=#ZWS zAF}(1y9`a_lT~+c_B5Bbu;w%khSl8{M~Q^nF%Q1>jyAWmyOhJ+_+6g=C3sVyRsE_& zuTt`;KDz6C5HM>XTQ<8;!pmCM&_Lv^Smj1eA91@;oY>=iPd0qr5imvl@u?Bba9?s`a_*g=Eg5FM zXU(rOrRPab#tYB)motHA*~aei1wfFAX`2*ORW57xa7X*{5WxnW=}@*_^0dCKy`33w zX*lmk>1kV$o)%>XoBlYxLN=$6Svahv7KAtblK%Xe^*83nZaQL5gz~le2L~bbW<~$; z{9j5@^$qaT&?4WN6>id9Sa9(XJ8gphD#sWam=G7V7&^s*-oc`(aM8qF4LKkfeMcW5 z2O%MCxK&lwlQf*U`&gC++S|Lo*p~-LuU0O4#|sS&DU?aVT#>IccS42F;nTc>rgXzL zW&D)>fO}F^wpe7A+%5pGYUc86c+>NpUSP;*dXIN{4Z7(-D00@~W2|6%EzB2QNL`d> z9)U^z{Wu|ZXKX^~++3lsnFCU2w(Lc0=wE^=JFL-{lWj-O2SMG|vlag^2iIF3Rx>j9 zxhq+XMkf_Zu@>%80zOj{np_Ef3xvH@i*+{dvK^}7)9h@*-5a%4;t~>QBU?3>yu!V4%16-~(SgtmR03#?B=X{g}8o^RM^C4lN}>G`1b^>u(L*0{htgAo48 zsBPObLnEW4{QQ&9=`#eJ3EkTcjnZD#COr(!y~x8bAYhQXw&b#R`49h#N@}Y&&mJr? zjg5_kavU;V-X$}H!3-TUOG{o7iQs!6E)`79vjpE_aMhR>IsoYYg6TnJZ5buzns=;)k+RPe<>BNpJePMVa_gzl3=ySkRUI4p*8! zz23-8rcw!58iRtn+rf-oLKOvMavt`KfQ?pSDk186U!w~7d+*HsYsu2s@b<1gT3GmR zRzeLa+F~(mWcS(DMb&|sd&NCcB*dET=`5-UJ*X{kc%+(Q)qez+8z3iB=*W#-e+8YT zM&1vF7*s9|T71-s=}~sV;S^w~%x_)EVP@-rNhVWhkLa49>6KI&ZAHkMfO-c-jKS5w}Y@zZ@sL8eM~ei?%h%iwE2U0hG|hDmTY;w~2D9n2ISJ2JZvteSt1 zblZf+xE?O)S3 zbz!X`Gq>5#g>h9+;IO%wkJzb3tK=h0;3~RD%>Om}R&z%~_2}}~^sUcBB@hZv*e2^| zRP|F^(>J4C80i%0kpUlX{LVU|zJrr|YC!a$9m~7o)UX1r7YUy3^plcPT(#j3n46XQ zi^ZL6=qHQ1yvcxOpojUe)PIlviFa} z!LYApGrXy}t_U$>tK4IZWr`ojvG*S!8U6FnyfJd8v=S;p+@) zf-UVj0Yk4YOmRtmZrM4#JnI9pX=Su`sSBeq4}2rT))q8)W)>dsphicr-C}YwC{W0I zm6RS<+d;=xW#FG!u^JZu#d^7es@gW6s#@Z{dTzWR4@zw~aalbw)}~0Mbp>~SSm$R6 zhUAndq%(d`@urn~2%5#l5Mflealt7cHSsqnt8)(7cQ{yG4gBA{roUcgR#~Z3iT7sO zq$Z}(Hd!nlegdhNMb850#xa{SAKPMUHkFL2v=bYlZG?)TY+}{|1KqbvaGx)rrTM1G zLY>&;AhO~jNPNc*F@uQtSjGmmi3dwA%|q{?bEJ)?e2+Z2xrPzk_#h>9X$PGm3vCP| zutrDsI;B)*j>%YabFLyqZfEUp{-Cw0jZM?!*;vbN`EYL-8>0LTqf@(>kn^BHMr?v1Jn6>nxbMO|tYhJzXy@J4r#mZmL5 zTWDHy9+2NOVU*^lekkSxNs3~p3e`9aw|bff5-R>N5T*NmE2!#$g7McxngOD0bJ� zBz!a8>ImCL0V=ftQk-&eq;d1`P0z0NG@6O_0;Z~q@;KXfM7a`;^6hTgMdvr#ZnLl< z6Nr*z-R^~4dIkSaadBo6H82!kth-7MDmaSVfm1m~L(O1x3{Dbvv7Um8ZFC0C36+eF zbF_JPcyk3V`H&;*YVaame^)XZKie9}1_OW8x88O4&G~PPUsiJ!WV$aN@Kb{yv!*J+ z4LwI59y#Vzv2Id{*|^=COjoq$1oqpfuW+bFEn6mmPMg?;<0LLy!QT4w);TDIy@0{7YWB6pTuB~ zLZxX-xR>AGSEwKS`6J*gU4qYt14QW0Ao<7Qn;U~^FuFy*rOA!L`kP2X#B~PF`!Wg4 zzQ?>bW}CRnS5FNmZnOn7Hs+ys(e0BGCMI{VzX;ZKK!!i&8fA4nj*)|bwY4LP^m}ay zlAMUy6r9rh5+`s8BtE*Vv&eg#1Tk`LtXl8LqYUy`jCDBZ-9D;7o1P;RNh0Cz zuOYZ4hgL3_uMZ+9e=%GK%EIqm@!z@*#_^4f59ACs#vQmf<|@>^#pvh`C=c%y`HCDT z`&eAhn6jDy2Tja59HQ~UnWS?_PdrtDb60I=VZankkr%&$OeBKG!9`VYCXrO3hRn>o zM^6OvdHfq_0gxPZ+3UfVSJC&RKrWoDFO(|BEZG4`W{|?9JeRloCZT?D(Pm&~r9pV2 z`!{gBA>k}EE2|Oi;0yEQ*7iby^67f&NeUB3q*)s;XTZsh$oyi{@b)&6rB0f{b3<+N+45;LE0N;c>UH3%%KB^E@%7o% zaB*{aEf>DFxdcbg%LTAB2mpZ$H*24>GU0;$7nfiMzJ0|9C7q zE#EWPF`i)pmNG2kWcaTl*A=QF_m5_`@)CCiUfX&^1k$PDC$Ptu5F=+1idZhs6TdER z_4%*M@c1d1XG%ZKF}xu<%lSCd_Pna0n|_FZ+E^YhSxw$G*gt?o8!|A!>_zr?Stw>i_kDV0z5+u56YEp(qhJS1p z4yiy-8_R(KBUrro4!L#guynU7lEMFq^W#a6m7}O5N5>< z?VRghw*=y?oU$@DNPNi2d(SpA!+yZ%nQVP7Yos;fP`{yJ9n{P?gvSp6=9=l@sR8jf z1UigiU1J^kCMLl)4_Hn$of8+0A^q71z|3U_&8{J-g4_(z_e(}nCUCQyynBveCZ-0V zfG_o4S~~`L`p~*)(wW8-z{5&PrQueD`N^qy)y-Fa917ol)4Q^@Ku#qyG@$L^bd09( zM8v!7y;K|xr>B^kn~R#NYoL`74zwR&BxoRi_`9>9H$v=b*^3vAcVF?=Be`~7ZX&6c zj4Yd_rP3oqHRPm>T%NHA@l+a(<{>4;shI$^wNJ(|Oc=m{Wm`ZL*~r#b&Bul828rvh zW#3neL!_vnfQ{{Wr11V7oPWX6n;@^C^4mbMIFkf#b*ms9GRL##ubRlIlE&0Yzam{pQ|vZ_^84LRSbM~I&L_;aV8A%0m5P?g$Rq)WQk9y@7agP}z;gYF^~M%@k@3iz57bjN zP)u(CD+YrKEj-$myF|=r;!?7#SuuUlPw0VW(1?|$JlU?UZc3VZmVg*417e?Qc3rG5 z_!J0-#RWQ$?V)>MAQ)lSOiUECeBu(G2N$na0mel;!d6DG1}Qnq;O4Sq>R`*kOjtZ# zB{Ipt-l$fYF{@p!vAt-M-+Q{o77|+w-(_akZ0kllefwwbkHs5F3=`tx_(KbvWWCQG z1r?$9%wT@r)8w=a(GqEyq+%V}GAe=nnUxxGv`#W_njZ zebH0Oj}H52>xf9ygc2j53R`$gq^RrE$CR-s1W9Ax>PFr82a`kt2hh&$p$`^!v z9SC?})&oTnAVN(QI|swDpOJJKhjG8!)Cg5D<39Cwk~FrKBh<4r4Z&3HI(}5W;8Jo# zDER8{zSJHdVwI>veJBlP-xBG*k=#e*!@g{_pdx^dj<<4f$neJ5+z~0?n6G)N?7EU5 zkMm5LybcZIMGZoE&K;K%_q25Ma3%V<@UD56&JZv_3lHK}<^s7@OgZzQ=*9fHod7dp z28KzYXOVAf+lQ^Y+gCOaTO_^q^uP#jVbcA*8g_T}tSv1iHa0RGpfT0ho(#2?w6jlx zs*28@Kgc*g2pk}`hq9i?RWsZFX^e6BU1hkg!*(cPp{`7?prGIdb(mJz&y&e+H8dHC z{4X|&Dcby8{iTNoOja_D?@X|0Fkf1<0!suL|5{{IBO_bcGMtQI9;b~YoQ_~W$N_by zK9-P?^F8$`MZcowi1VHqFGoB`}bIpeLNz|mAV$%mA3i?SIkf?!mSi!g%?0!#3`(?tQ1X)~M z3u-`IoUGG&93K%1l|M2ATS~|*G0}s?);?T_>iJQ2MQ~ljYfWcoel9`GrRrPLs&w5jfc- zq&LDp6}$uh5HFn*hH6*d<`!=%=JA%NPeCCo;S?s5tVF1{k;yan(UkRW{n$7&y2>-Z zu&`>t=#cAzaQ3_L^9E8N*OXC)IIQ z`n1+Tg~z{dN1l5sj$@Z81}5vT3Ob{UyuF8?Hoa`&5IQj!5Y5Bz1yaTn$#!#-)X5@~{X=qy)?CwHUX&lBT;Q z6z1)~lV?{my6X+<3!$%In6kYcpy2P?3Ey*LEAltckv2=gYB74oG2r8EIcXC7ELemU z@^ag#y+-du?);l%e^f_^Wu&ozSV9KcwxpjVqW8`)Bfh)K(#%Kc9(VFQfi2rxz!z#s zj*n;G1iJe_06vc3^vuv&f>k`7*$G4GYzOPcq2ppcOl(ElU>i+jHFNc&$$!n9>#ztMB zEZ}RgSPRIw@*tI!`wlb&YBAoo=&t>KbD-Sac82||s06Og(&tzpO66}POfhJC8T<_5 zH*lG}C?()9RV(kg&cSpAcABtMo?s!^v9;1i&9f6@BHLWE9n?cop^Dx)$!0yhy996} z{CTRL2zhVZdpo}}sY2rx>S-f%veEDkK@IR0L>rq^RZJ%>?|Geni)Q)mT_TO_%T-KH zT2+O{U3edHz*@`w_FzBkd#|+RlBHg72ncr|$aW7Jb-4xJ6^Mxg7o$9?otWs#l9>f` z5h0Ey*4t;847+?4Q^en(#Q4062D4PiV$&mYZlR_%C$H)I%}xHq3%VQ*iBXT1Y?Ugh zIi3=u9u59~l_!+QeTTjK3GRr}@ZN*Bkhq5MT8CmFS)kw1Hn0*Fe!|u(oF){$`7YNA zn`gesjbfOpuNQQk3e3vV5YD}ql$n`tkI4xQZTPy70?r+*qq*;{;J0LH zSSQLt<{EEfP9Wxx*yJva$Ku<;!~u^>#_&bYv084;^ds?{b}%;#S_e!jYF|A1)I?ku zqFiL@adq7A%&jfU&1Hu?UjDGj;V{;0FK_8VVuXbjM$>m_)x+?VE!deEQs+HvL(B*x z>4d4Vf*=tBq8|}do1ZrolHfXp7dbGM+AT}>JSH5GlcddE|S6G;@TwJ{R zH2jopSXz5$IJGR-%9o}-^fvvPB#{w0XjD~C4{cIfSlN_q zX)F$s{?;3~KzL%Dam;adWK5Ln95=|xrI+l%Vd3)2EmMKyKwr=i& zAr%m)oMK2UZiVvSC2?~jE0l#tq=M+D%Aamq2{0!?iO41~;!;K>kPOH{KDKcvtvy4NPAw2WJJ(`NU~lR$zkp zNeFCzOsH-owNm8j&xzb=LA`PlqE12NFazFYm^fVlCs+>DYJeuNbLf9`^K z*AV~n^A`TVWxcZJN55&59v5Mj-M5cU@%+y?1j1UCL}Zk<01PPz)4_oJ-$mD>fk-nD zgz-mgE;WN7BDLH^HaBj9QQ=&`*8h?9OH1lt|Cg;SuBed|9Mi7fJU1Qme4$MV10yNyr$?2Kc! zTRn;(1X->_+d!>m|9pi{kAMFz}^VX^2<>;B=H%5ny6p1 zss-k?->(7q*<0}|ev!5&KTz%MpLAFUCSLrQFY+StU(`?FuTnEI<*wxE zTm%5Cj(q%Y0B1Ci-3NnP~KEdklqm{4iR5so~G$Rn5 zbvngRM=O?WIII%Qda7P{<+5O4B2{Wt%*%}-imB!hb68hdx~{EjX6}0*Y^bEv20o}=lfWUz znE&*Ue1$54pt8VRi&Yk)au|)msq|NFa~LN8hfvZbeAM*WUD%uTg}BNIy}cSvV#W_c z8`cu76IfaWqk^6qHjYkF*___Tr`q24=>aK6{J_BHuE0bL`*&&9_bP~8Df)**Dkur{ z)zj_L<~VSr55BY#D!~w$DJ;y(LVBNCnz8h~!X3&Mds~tbe+1O$)23t>_Y|=w794V> zJTJ_b?zm=dc+5$YmKtLtYi7J)bk2cxqU&UP*k;-3t-Wt+-HLsSd<=L7sG?BiS-Q5M zWqV$yuR^G^p>a1ggFA?dKTuP#mtkrZ030*tCG}VDOEW~50uz%bx&i`jGfWk~_Zg)+ zQ_0pWD}Y%}sl07n6~~0JE*`8nW%XgkxR(1zNP|Bvj~!8{D0uB753UIKl*+@0`;0JB%;N%LygF z*|8ve3&%U$5esWi#F>2xDgESJA(mi(LE~pxk#T;uwZ0FONXkKCs5+WnQ>{KDC)rnb zJTudwGv5>7+@wON#GT8{T{pIvs?i&Mqv} zEPwS0Nn()d6v#wB1jM*ykLTOhPSs&t_UZQ(sCjjV0C+qkWM^#x$3pqZDm-99#RIl@?soW zm)W_s?kz}nYsmBMcnOGdL0XFhw>{VwvV#y|UPgQOo%G4M-e@iL9OZa;d$sHlp^C+* zBC6ODGRSY2b-ec2+1tlkU2xeX%za&ubLxK(KFzReiJ_j9RQnRIdXh`Q`$i$36aXo9 zQdic?+$Km3Rsh9g8~}U{)@uW`H)9Tp3Gm_|yz1C@d>E+!YOZNGT#LC0CslZr4Cd|0 z)U_p2rt06sM^C;yuCCq$(xdUAnbg$OB6o7~Vk~I_K6%GiSo|>1AF!S25DaoQ847CHD3Kz;q$-&M#5CZx% z$+z`@NRx5ZOzE=8_gEhFAJ$X`fYzYv$rgcl-9=qp5?19ssV^=~mDH{`Mz2)NnAkgVhTAJ(baOiYO z#{}aH$-XZ6w7f4p&KM@@F}f>C7j*t?XmALM@{Ur4LV|5jv;Y*%FYaV(2ZD}~O|6E+ zv_1;D#@R`a8q2ZIaMH_6P!!YN%+INJ#;>z6S*Any3Y5rMDLFmz288SLhGm=*V%@kT zeJRnxBjE|O+^(^Vh^`vzpiH&omb}(f0;Ti(1nJ6hGO!z)pW0daW%g1+%x3 zOH=@SGa+xd>MMmys_1S^=RR_+stX91>RCNx$984VH07XzoEJ-Q4T%v!Ei_U6n$|jc zGj>&VFb60@6KOf6dYHx2-=^3@?N z$Ohu4OL+$XV*6JgsP7SZsA2(#qLJ`&Ds_>ygS^4HHOTea@fA=Bc#uYsgqKiG zN#eX43&0uzMwhQoeF^majA%sB7%Ue6H8Bd0+YIq0k*hne@Ws=PgZ;Ynr5o&*{Xjq% zi9{J+Pn_=chf0T_{WAoR9POZ~=i=_FtX3>l6=Awn$E2iy3+O8c;c~$G5xhAGPV*2> zjCa^Y^W74nVmDXZUG4TsNE8l#$<>2JhQ>I6RM1rk*zX`7ygZa)g!|Xj^%_E|OWBIK zd^wI6Y#)AQVeuUvFTC*uMU&q6KF65jazg!mp$NNfSedjBRLrTGr6MKfm4WVaadlO4 zaGwa|mW@Hm(&*|kW_<}`ZRgO1co?kZPa%>x!D6Zp`&F)M3+kW?M#+XsXxR6#>bbWU+GCt|6o-N+hP zT++-MPbci@Xldcv91Snu!sd0uQhX4Bzn7al2p5udfDiLF%6nR`B_e@VE9UUu6aSzm z2UY4Fh0q+}y8tJa-se1B1Sc^7%kf<)=$kZulqxYCOY@|DZyNzV>^j*U%A<{s$3U$D zevoRx2y!N1jOrR%+AVI;k|1{-p=%rG?LmzNMdCZtQMk={M1D#QQto<&jD&P=-!LHP ziIh?}feDhRLMNQQufNG0YgooI1oP zLm~Sh9Ekk*3DVmvEzwjI4yyQUn4N>M00EVST1s3?O1T$v%L&5++8!`V3_5Ia%4`XDh6x00&{-_*DYchA8 z8+9tKs6ZiS^|Fxk)jliYFhGXR7|uR-;{v-0CF>3W;2PeuTDXv0P+WXCE-sC=wTL60 za}ExEXn?e@inY@TEi=YeVn_`6A&UQSFmeGlZ5GYMiV(f)qONm;MP9&R!ThX(92~7y z-82nYcfdRy;XE3o7ug2oh%&lmP6}6z2`W|A)?S0tBt(IdZlqFXQh}~nTgM)T!=m8G zkKpcZozW{sP##9M$Xx~w)oJBY{cMe>%H_TPM5@zHe#t$wUgx;d+iM63EpcaK$2YKP zCaeO#+>Ft0S89QE@&T0y@)Y1g&77=HLS>e7REq^HhtL(vOP0WElUI}>uW*^?6s*jk$T0Wo2g8Y*;TvDCHFtC;@YT6t2QKIEYCY)k)SjE8-B*{#vy% z6nv`$RVl1f<@~qJ*q&Im6gb#Qitp13$#|yV`9w!g3l|1Uv_@vrr`CHyw0mC@I-{Rj zh>43|)^zV0BKG!lJm2W}YF(jclKr8$*br1~@4j3BxjU4-gR-NpnM#oRkp$HX!3_kZ z!E|9{SIP0hX1GoX3uMPR^E|7N$abj#>Vt zHOWj}#<0FN5=o^ET;wtzaaYP3Fi^>m1%W7fbGn(UFku~#<#74S7sk$3?J#7-QJi4` z0XF$=9`Cq}4~g9wx`z}Ug3W4Z29i&XtdDtHiK(mmm)PFdPTvu(hs4H$NR%AQaTrTT zEiRUd75&GurseVybb&8^fIsYE-c!)r-<6b?x}lH;TH_>roomJRs#U|Mg*of38OL;L zjM`n@3S??bFYUdYfA1;#x}%(ythJq;Q<*79-cEeFyOmjf40KRG35^%L@Wi6jEo5aY z+DBlAe`!msn;S;J0lWC~3!{NsaDV{P*nfdlNXUxZ{f8W8ON@dn5%_!E1c322eZnSD zi2z&HTfkQTmfyJ@AOm&vBjT+-aQ4UZ&p?n5e=3Q4U;Kmzx5O*|!4S43Grxd~&Mn|# zTL%NL0)JCWeuw^t_x*u#{Rh-IgtW{8z!y|k@qcUXx}%!R+C1Qg?1wX=(nY~Gg22FF zqZdWNHXt?9zX1e>7DEXo5aKwabg(d3AVfMMVIUw92#f;(L}>~LN)jbVm4whj31sj4 z283_Vw|jQ?>>rzRJV)O8+^65)^St+73lsoE1%?176mkfCA4&v8!9XvVJu+ZX!;NMc z&m;(InI5gMM}i(9_>wh%@3XM-I)h-WYJ+OHgKGN!WW;Om0hn7n2kl6LCOq9q7>|7N zmKaCq=DrF=0N)R_ITWEmPo*}LV@*S)3Y@y$_mh&J^knHb1fqp0R{)|; zgfSZd!UpidxJ{i;NT3;75O3B74&h3I(*m3+xc3BvY(CngENud0bF%_~zOG%6IE5kx zAs`6C6&nQ;MHr@oUm}nS32 zoX9jUn3)g-`LMe{<1oDA8PUd)NAvB2U*FlgyzHGTM7XX~?}`ID3^kKMU1kOQ?>%}X z!N-6vquPa{i}7>|t9?uN3g8%o=fz6^aD z!{wp&y6J>r^FHLk0+93T>0TJEss)qw#!by++UrkW(-}xq zn@lywVAHq3Az)zp?*DB)7Dz}z*zoC;CjVylc8W#FeJS(DnV-~{5LE?Nw_O- z6WLJLZObM-g`mg(>Ftjo#QUrMK-_GB4nb{lFytkG*f$hW8Foex?Ef!~LmjFMz)l5n zjgWeP-#!Y2)TfYZpAOxA9$vE65o{C)F8JePMjFI8+es;`x zNpz|@z@mXZ<|)N>H4+jUnT6VS|81I`lX7sigFn$W!Jo}Ku$sl!r*J&`N5Gn(&3rjd zLCnMXb07DB=EOLw!=S@c9cB8BP{vzr(1cwaBVAUR2Au^Uzd)ccQT!MT_l*J~QZbyj zpSoP4QEL-amCJf&i2lcr$m;|B;dcX!UU;G%ydIU*T#rigFK_XlI{c@Q{H5Vxc~Aeo zYXb)o(#afa&)ZVE+wSc8PUfZRSgIZS6I*Qdw0-rg_(vzb5raA70x_@>r}2tSJwn*! zTCsLZA`M`~N3=juBXV}ajf)ZO6s1adHo@yFe|(dI?Pg008Cb4|mY zWiyI}!hex6PL;m_oWi*{u!HW6<$0$DC7gM|!u8jQvlt)<6A707{;m5Y8CV65b_}y$ zW#t-nk&yiz5&{>&oX1j7SG|unFM~gGiG@w zf0{w(g9PHBYyB5Sq&0^Yw&E%rbr>Qu5cq<7&*3sqE_kUNg!x>6p3!o!xmrLOknqw( z-C%PGo#2NC&NH`B8%EjJ&!3BvgaXSs&Xx!AV_d)toZ>eN9}Bt$C}D9CsHmu=7o`eq z6gkCq{>K0xAC>su`mve^B{7&~)Y3dY;gW8bp25;bP;)Pl+a0-Ew$sZy-@|jrp+23{ z?|93+D^Q7xkxv<9X1&YFL3b*S~V)<>lk} z3*N>Tmj;KxdVPGwl7@m{t7(t=MH+jV@~b)6OlVa{X87WYYm@SFJ%9cYXcu8|k1pkx zSME(KDVX8LEK(D~SEj|F3oxWIdH(zaV;&abCTDDeY=;?B1mbxn>R}=LNPuRxz zCQGhHB|U%MreOJsgEBKFK_obV_F$6>TSKO1%BYLh8u+3Y80CV3GMj)|jJ!nl587b= ziZS}Lgl3EVVtPaQU9||TL!fs@BYX-RSZ%3cxsoVc9v-C_x(d=sR`>Yo!gJNW_ zDo9bIB%yK<8F_`RBTM#zIRP2szLY&9r#CD?C^dUIUa2bb2C*N)bMGwILCMk)>ve1S z&H`#QKKRufAR|aMPY&LHY%53$Qoj$$dlW+2U<*}&EppwjSP#kr!iF|TlKT25#Dj>zUqN0Xbp*Tb7;LX+;I3}n z_TSdbTQC6fggw$%X*6yxq(7V=L1B1$l*n{$e%>j$Mg|p-0aGiwgLE90>KBmso{NJNJk=ahb;tqaHiBmW zU765%r=>9q{qgnvPLxw#&@C;JjGLf^{A}Rd)(3(SZ8*k0@a)uG5M#db0Qf~h{nxQ< z3~k+;13^G8^?!F~$U79#!+8|;UF)P5gJu1-1p))Sje!^eE9fpNB#Qt2yZ);?zxn5v z4A~>sWVay<0S#YS18EERjcWd9QvVNkDqF;_n-`=q7g*4|a^Kkvxov1cGNgC{>s&uQ z6n7T{AL|eVhbbo;2|XGpb0Q=rNST4n;|2V$g1+5I*&J*)vrHKp1FY+tm3xCP|DC%> zi7!$%BnIndPEFuk{=)TtNdho)$i7C2gUw&{QJ_OCz4p*`Ij@ZcvcgR^2Ac%L3kS=+%x@aD2-r>hOEv2FYOI_5!6;EvN4M!W*sx7548W}d z?`EY`oiFK1bj*?et-Ak;imZDeI^TF3le9-E>r5p*Hq-H#h4Jp8Z@)7v;$53Wekk$5 z^7-zbQAZCC4}18NcW4d$;^W)i(ViXM89N_%g084EhtcgPKVw)}GMbQB?k%vqMp96b z+_ul|iHQai%)tfQHX5amcv4Sdx_l|o?rU(`;@Bm%Ettb*l*UZeXJ%%aSJD%GDe^XV zCChZ&;@wdiH#470RFFudLTymUEs(ioUWE{){xxursOm&ImMNwW9_s^k_30b+xp(om zEB7^Stxudlepiuu^~B zbh=uKbj&8&aX-x$VpnghFJ)&z0aCxt3#4{vxBF10y5|M>W)MI7b7A=pZt+>X(TN2b zo8O7L(y3<<&xont8FQtycDac?)EKU0-x|OoV(5=DGGgCp*2e}Np>4a%WO{)0m3iiz zM~qdYhcy^&EPr9>kgAiuRsZ`q_n%GaY{nC)rB|q zTxDBg=ie^0A!C%l5_H;FHt&o2drYMdy9W|sllO>6l$q%9b3lHB;iZb^m>vxBZ56p( zicyPQJg0_{GhSM=#rjOtH~?gW{tRf)DcvOaWV zqBZYy)QsIt<5I{9EbTySdif`P{CoGS5nA@Y%jVF1pdC&9*|tc7k&!Q74A4*J`igoz ze1i_iv{F^l#8nAKy88MFj?voa@gyCC+7)}#hK7c+ip$BBJ)d(gW=)6(lVDDu!o#H) zrHJhEY8pq^Sjmmxtdc&EUufCh(UDVK6NAzUVt;HOA>51qExeArCt=&?M&u0or>H1( z?2=L)sFAIx_NqN9bL}60?rr(u7V7<~kz>!C+@tTQjRR2z3MV|->2GuFE4WX(jQS{* zEU+~*9v?rV_NuJB_I^S~`lnBy7KV${IX%Ui?!h&?0qR?|yNFmKS(D9xZic+`(k#^Dp9~ zJDtDn4)dMUpTIwkRY{BIM1|^tnTCY+Wdtygs9(TjO2R$F^H5!QnT%WQm4rwS?x4uk zX7SN4gR7j%re*O$9odGu{s%?$as1*HCzsYAX>XH0J$`(}!7(u{kE*XmNJ#KhGTP>Y zauvN-$(22*P*xukMzhh7b`SW@IMlKVs3vdG&#EaVYzqb-LBovbi?2+iFE~SY{N!0hPgALt=nFDq_ zgU20VEx*7y*P-i!1$V7c8W*V5cs0UrL0b8T4WIIYE4HV&*Epu!Bp3K8*_+79svwJT zz;Mf|U6QK%#}c;PQzdk~aiWo=IuPl>PTY_QGw{m9$;=O`1jmXnhVeEZ|NI@Msszh? z));M0yf8F8T-UR{Mr5~wLQ;fur?2|2B;U~O-+sX_RDcgd?;X-xs1C{zA=jEj z%y(%U7!z^Buc#`;I5OIbd)%X-H?|??gVLc@+X;N2i>=AFvhs?~v!WGBZL}3R;26QK zdZOK1LM_ool_I;dGE;7Ana&4hK664xEjf$t5@o3ky3Fmq<760Dm8+=W3{>CJ|vWXLgM4A7Uf? zBXz;wK{O;`hD6e#mr}a|4e9s+SFVAAV)A#PmX@|A{WV6H86S+F&xM-qw9=6E2*{HN zwN!@Pk{QTmOTn{}hFLX`09doOT=n+m_pL!v+ceCuPm-lOd(jygg*bB8B(){>71PU}aR*tRunWAtz;Q(sn)tTSL3#AJtd(VmhiRM zi7LSYaH!12?+k_hrJUsuIc<6iC?D>brsid|jc{9fg4RW2wSC|tZX2iF%Ptl;7UMZ5iIGr;tQyA2=gR#Z$*`%JwO zfe;QbyBh2y+I^}6j3G$Bbv*Sza%T&`3fkHEDd2oI+kua&$Bm~WHS4TPG9GO$HriWio$!a;TNZYQt=tQEEU z(RqXkrPr{-@LUvO)>e3O(ma@*E#jym>rrQ$*ws~G@s9D%aIWtIlD7CBWKY0BxSs-S z28s)U#4D!&`coVo*yS|`;Ur605A`nb(V;@=NyAXfC;5k!H3KUn9E3{{Cv&=LOGMts zwc4K;6ITfnn1HRL`5k~0bsN3_K5Ig{vuPMz+uvd-+U>j7)u5)JteEvO4zOUZ)vH?} zQ;kr(?OyP)w``5e+o!Lcq1`pkIe3smO{iQLq^?MJcDa6^OjIR2uug1~5hII?SF;>k zl0m>RcivA~$`JQ?Y|vNr{!0n#2z90>@4jUoYphXA!?RT|Da~Ejau7W1#z-;v2Xp}n zJTze378Z@Ci{w7P`Dkl0jxYFhLws~}bXw%IeUNl#&l0-Zu=-uFs4YXT1s-xUNM^#U zxCU-6$gcZ~8x9zM#910LARyTIeJ(c{Px48ByOIouZvo-1v1$yku9v$>U)Bumc;9OMw1k)&R=i)24BQ=c-eQg=?&Ci_ z=wK541lQg{H^bBuSrNbJ*_ouZP3nwY7CsTi&91JI^~lN5a$~uPhS(jl^J^=x;)V>$ z$sWB5P4k9Y2?!G;1An%p%Y{a&)lu>r$2%*GKNE?MrBPP;qKCu_xnyq;PJ_`}cO+^{ zajrBM;k`;tr{;dF7+26b>5|AXe}c0}-VQQ6B5VHvqMb=;eInSv1Ed=9W;4U7D+@zR z@xmP>(lH5FVM;0HVc7QW=H_NPP2-p>#Id#hGd9!HTxK#)C7 z4o?l&>b37SoXxk*NC26Pg@vW28ll8zFMUsWmZDPAm>m@A;`-aAJIA_CRd9V#QGY0m zj4K}Py$GdAacz?|Y6SH1f;)l88p8t&5!T%!sNyOyFi$co`$55&H*RNpvWnSe_PJer z6iRH<#mg?MHuguHDWS*uu6X^1qRr0Q)=uq19ld5J9K8Zk*;_c@wwR!0~GtbDhf+15lYIccm&cP_;dZ+`h~y&oXweMKkS`5))W$_L!*qC#t4 zPn)iWzR6y=eo20nWduotoJ20Txc->jGn|=G5O~1+I-JzC=wQ-u^OncYKkvsE zV@63~AP86i^KXj$JWRsT4QrrusTEH%<%7vQkUTY|4l6y8HVHJ*thgaH8ttVA^uNb& zwx@e&sK#430Es}B;56_w(jGsxi{tn}0J^L(J41?V_fz0W@nCkAXoXA3*)xrAPCL*V z{(O*eKep60THcL9F2t!m`oNi={GcM^)noWg#_ij;L16NY;tYaV3fx-`T0cMkux4O- z5$h)$SPOwxh?}36a2HH~JnEdsxx3y5yK$KI$aM$zqQeAaZ>PkGs{78puoL~;k>^iv z3zw2n`)$nfV7zY@pz=qr!U$fJ0zl^k{SUGF7nzWi)_(hhSK=sG4n$Oh!G1!MoI5 zFkFSSolHF<@P!aAI6W+=TPYu0aRpR%-Z1(Wi(Pyr76JM-ram8Txexh;p_f}Sq++-< zY798IZ=K9Fa5`zjWUawRHgpy2KATM>aKJ;>oXd5Kc>bWK(HMWL(4mi({0fE|4zqrB z?$C8S2Na6<`;gm&X8T{I?~Iv4thkB}n1VqIm}Jm-p3@Pd$SHO^%UjGcfBib@4IpyM zP;u#jRTldIsIFp5|3)SNlS>j525%ZMKcpHjo^Lef2jBDq4mJ>a8zha?T@bW5)D}(M z03;y-K+ud1W@r%s-fby>$l(=gtav8*YgI!j!06oow{3-yRv6C*OZj^G<}#ormB?$1 zk0Ui~Bi`KH-JulA`!y?+Tal9lily=DB|mmY)>w3H3U{ny`J4BS*IztdcMkZd<$!tw zXmF96ePP>bT3ImVZA9fpJgvmO?vz=}_p8|e7Bt}!`8DH=D{{{9>tzX)mADYD0BFWq zg+WRj6fT8?3^b?`vyCu~yjcJcLWhWXdDv2+I|2lF@l}1|0#$Q~0+yf&2TLpOECq#G z=~O9LGuo;cW;~{`8IL!z;tC2u22W!aZpFsMVM|5s91X#;#{K{-tWA-a0M$YwL*dtV z+n1IU$188t)w#Ork4|6;|7Qn2yjtchH1CXAuG$HtB}9QBTj(ByUBkbwlq7TjyAv({WwKzjO{f}S%9mEVZPyhe` literal 0 HcmV?d00001 diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram.png b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..d715406722c72a68fb6d5a11c5202a56d0486db8 GIT binary patch literal 55849 zcmd?QXE>be+b%9uA|zQsqKhD+_g+JyM2+4>)X{qzgG8{Rml)BbcN1-x(Gn8EU@#b6 zL>Yr2dS^d=E7sm$_CDU@-TTx3KOdB7&wXFzywCHxuIIg$h7uVG4G9qu5t)keGi@TG z>u-pNt`Oh24m`6DA=b;^6))<_l0+v?q0l0j^xqR%NPGBKYb^6@i!6s58Ld5K085SqLoRA!z)Hd zoUEADlcpXugl%t<=f-mn^-QLP1(X|DL$mka%h}y%hdUq5pKsLG!V>kcI-Qm>VBxuu zqwELoPcMAQP6J4-W8AgG4Z5u`G5Co7gr;3RbbD6{dnRm0pXprs%5WLX*vJk zhvyc-W5%y>{^5=P`QC^cfveTrsw1806vwlzD3@Z!<6AW0QU<2nxX@tkB3aMhFyf2d zxJ4e4UnX;wjJ9Fl>_Cy6{*)|YBT8hM(C;N)+1`4PK0I!jk>&XBcW2ltU~gJakih(u zaoW~U!U;T2k9<;r5YZY*4w+Ic=w zgsBF?)Wu(FEnnwf_qi>uXtKpS#yjAX5bl}st%n;RvXEuzgTsyJU>zOf78a0ZW<_=j zXoay-WZpX)VfGX-yv+i*)*sa=VLbC`anZK!+GUc6HUYZZ1w}{m=pJ2ts0AzeWBb00 z)b8G{1j`J>I$jMka$oCP^t0^73?3Wn{5zJ^%)-AR2vt+u|ttUNFdDqU~ttrW`=nAJHq zlAK~u8pUH+zPifQ_DSYrVF}~O_5%*sD7mxW;)XvVAyBS> zpg=$-s+vn!SXe3B&gMYYUGP6TL zgt=rZdfQ9ft|tNamA8xE!>itEGSLa6yPy1k!ss-5l{vyDz0e`>`{DRo3yzq+Psn{P zDKWA*TJ+6Dx1OTUieKhM5;yG%>hVzx!Gcac?ygpRV4=LtSm_!_$d++`O7+j?LWVp7 zB;U|1Q=XnMt7N?TA--aYSVvk^;spz2Tz=4K6iJCra-bK0^z8N+mfp$;5}KO{APb3| zFmiS@3(HY0VtqMHB&e=~=01y(G2F1Mzoez8h_vk!QzB=JHc$*ePP(M1VRV(Awz11Dn0F2;PlAV#3ePAV3g1xn-7LC*1-`f4=m-MFZah0_5zi1TQJ zktaH{f)>`Q2m)Pw`^O>i%NmY1D5h)87jr_Zh{;Sml*8Mh60|1PCgIk!9^8iIwXJ8! z=d2%7_C3NLhsT(7sfDd70h0aNc#wCx&U@Bcp_1t^`)novC*8cYcI4g(k%9G@ z@L6Od53VDo))>&OYiR}J)!yk@kkjYGSkWKVT>j1G(tc+}H$(b73PGS=q<6qETC%C7RGuYojC^u|!_) z9zF$&X2ruk50jiV1O2GKZm77f$5*C}#qIe<`Ve?9j<8lr-@xv9734w3ijYswlFZ%? zKdFMJZSPQi;`@M;nEE=GcBR9fW?xYEeBxIQ+M+kk69I&n`t8Mu^$t3;vL^QX1MZQ1 z_cLOOWetvmcVTdr+JK9x3PwWra*45XqL}i1Od9e7I`=o#N?9*c(LoaCj zfXh5zR(d0Pq|yTBCiW-=RiMI3elz6H>K}d=*Ff7ijY(!{Epk(KA37ObzNE#){8_!S zLUn-)BBzMz$cMGOYRXY9WPvWPRwl&4A4A^jRR3z%TNYy{qA}-F+ne+U4_?wTxGHDY z8af=Y-KNT`lr5R5Vdj5ebJ*B=pdEKf%Xba2A~NNPC3uL@4TN0qfcS9@1*EwohdHtBXm33^Yt~k_+OTTEIBDpaqAW zEj;dqJ2xMlMlQ8*C(ipc?j-XDzw}2SLM@2;4B^fzV-L)A9arA#di0f3pNGbwDa~)| za7+7vwrd=zd{5iM7w&kK{4qp`VE<2_%c%qp-5`a&W{c!WiBRiVS3A=drArvqPkAJT-{A$&Udz$Zq{p9{ z8t);Nh<@rkp$}l=RDn&vY(a{5aiPfJ_|i*eZ=7?z{r1wprr)Fr8j=ArI*;rBgwd4W zNu3>#lL^Ytlwui%H&e_9Mw}(B)7Xqx6=S$YxJAuiH|IJzxqC5I_BN^x&f)_L^IQH$ zfx%E4_PaNh_OqiW2#MGgf_oDvO+r_v%m&@^sX9`6AijHjZ>4YrHZY0soL5&>{cU#T zA5(uBv*}-ZBDri4zx6X1^ySK@CnkMq7ATGeofovJ{nUbr64pP|8ggG59@al_f^0oj z5D)#_tnq|GTfs`Fhrb9@0^}F*>Fb+(+^U>1rUiS`XIc>}%fH%6lv#4US417A4r6FU zeWH1gZdH9I{l2ekjG+4ErWJSe%XN2h(i*Um@=VU-$5kTEr-pe-F+(;f=K0TWJ=h-G zxJMGUy$0s2^*IUu_)u|~)WK(!YuMNKL^e|pQtHyQMguvEkbQq$GjTfx7OeOE`Ev+_ ziyt5Z&4=UZ#;qI7!dcr>Dubt2h<8F6Pt=$rBfU1la4Wx}c$N(Mb%Sfo)zy0kaNn6D zg#w0(^KsOq6TWMMWw7QFY4BcYV|dyX;<6n~wk!}FbY?zX&ymN2QH>`tUROg8$3?+}rm0n~< zYM7zmv;AP*s(OJOagMUxl4^%+j@{K^Y2WD)tGg5=5j#I%jF{cb#(QbLXKz9e+vhYL zR^XWdOK&t$_*EG>fa(PYd8Ce1&h)PY0?<3Lo5?X{cAuPRN9l8(LfrA9@34xhpCHb`Kut>3AB(GOf1!iPDg-d{2CB z=!ILMJtHl|yGev@J?lYNR*QABD{P)r@X6SDd2*A%JkEThJm;7$#?gnxg>aFz(YOAd zy#ZH-6G$6?=4IIOBqn^Nq)SX=mup(# z>3$Jgos|J-aLZ+SvnnevAa$+zD7FDRJ`s9|Nz!zjQ%Wb#;-!U!GbnZ)ULx7@r&AJ(+1*i)5};l!OGt7#51 zxE+zB{B5XmaH@2)MwCViYOp$-q$yhlUT>NEuq*gN*JeRRdUImt`&BuaY)u^Z$D;Cb zF6wY5lz^Uo3)>;2(!5av#DM;UJ=fpok0MM+gdBYAF>uHvZwt6}?(XojjQQ}Uqa>vS z>{`;ih`fGDz)7i9^d3Zx9=5+wI18{k;~7hejn$g0di(bH9op5wVX)Jro(*CKq_#h< zFE(7xwQ0j=QYzf?i71;=@X7ZG;n&Y_iJ;*KhwUh(pfYBqSjA3&xhI!lOWpR}xDlJx zAH|{4d$}S`lN7!ue?1R7oK(0|TwJULcP?|72|D?eFjI;^u-Z=FF{l_=8ps3EGRmqi zcs-QT{ZhOo)j3$4VnefEn!8l8fKX89!>PGoeZEn!sIrBV$vI9suI$pgW2IcbN=^Bq z%{(ONtDQ|8M=G(5XUDoiMDL$H3J4h6I{%uEINJ2z!dD1yKCY-yW$zz!pyO=V7)N) zuxy?!vggEbHjJ1;!nda9rtF7Ih|#3mL~_yZ4{SDjf8?#YYaM5$!j8j}fw{X1fSU#iM?XF0=2rKe3My;a1*Ai6Zrf5Fz@YNZ z_IE|SQ0mxDLUzsZ_Z07jHRWx8lZT%uISo3ayLx(tq^P4EL6)XKU!LRxuIBuIBtk%Cr0tcqbM1J5T-ob&6^%mUhTsC zofDw|Cuk8oJSO-~>nJE#^?WRYnpWO=z`HE6wBg^3+}hP-mz6!O(a`ljU;y+YBV^4@ z4DgYtecx)TyOla-D9w9CFIYRqq<&pX=}F5Lr~8$o3Du6T%L=9C@*-`( zB=K^?wXH>m=}GYcT!pjqc*4(Vt&^Rv#AUgycI|v#GgH1Cl9FYe8{tfu^=>UZzEj|@T zV9W08H-II#>_%5WW2ZZf;kBh|Y2pq)D`j&Vly65cVWiwcgO6e}tGPIH`rfs*$(z;9 zlZ-*GUVS6is1GxGK$v$EquUHB z{Nc}bJ^ndQPYX1vi0_Bv= zBf+B#EN;u8?VbC;U1ja%reVR|`Ql&k^1qLz0(0TP!pkNazUG>zc z>)U(4od7bI>BEKrT;-%7K~?_>Rsi(K*CqsV4sP-xE6eZixvlR!XK*VV4`rM(S3i|K zK+3Ai$=^)>dX8glYMTD>>-B=Q9o`-i%JMxBcBKMfF%l9Q`3OPs8}xBoM}m4{O>$t) zvg>(5e~d_V?@9w*26vbPYvVfK_wEc{#Nlmm7L`7(vD6w#tTPQuidury&2N_K1W269 z$PU(&-gh)9TigiFuIZ^p%>?OIzY8v+DfbjhF7xO)u3B4y2 zkrSv+A@%0pzfgxM!6)$3?Ri0x>yoc!EZX^mO007$8y7hsdnOiGyY*HBKsTrmROk2X z*eg&nA_WCDR}Rya6?Hs4d+G!Q29qRYi%>G^J7i>6^s!yB#Om61{i&G?n){}}^?G`u zzdrdlfqTAqUmk;6*F8ySR&`qU>so@yCScf^)j4_uYFk_O#+jiNGL0#TUSkCh@L?(D zu{YI6DrSnyQp|Irqu(IdPx`=&-zL9lqLmG7ZNo3W+5I>dZutYl>bIn7Y(*)sS#!+ghfQyMMaBdJa-Mm=q9F8)IPGRFr+l1OlQp%v(iQpG4G(dtKF8Q zOZP#4{L43geYessH|E8W9Rj*gq1nOi_{s6FkfG#Fr)9*hB85;Q>OQ={3?--?# zVu&;4F{o$K$VEi~CsjL8Boz`_9Cz_G{pjZ5CgNaSbzYZfIdZkYrnGL0FW_*Q+C(hg zYgR(l`_FKL33*C&&YgWaX*I1H<{i9a4~q~B-`3IDV|$4q%~jlGokTp6lAMR5)m+y? zzq)jKcUE%&AXTJ$p#w5N*ZHmrSW5=JDAp;Q`|`!ihv*&i;4HMIQ=Ay|92v!9O80=K zeCBt8dBDN24oTKI;Wn@c=3F$iRSdu1lg(BlSi51DWhj7x6B=F>vCX1ai`EP&KB3<{duuLRTY6aqBe+q)}5&fL{F zZb~$)Jz~YR^_cYpBOCZ!bjr=Sw(rd=s&JRqoy8^33HZ+BEnfD!RPTgu9>pgmk#Dt> z?y{Zk(cX^m-nNHq`&(LC##uG4Ne(5+fx(lTft@i7!W_m$06MyTLQc3(VX0q2Ads0W ze-87XTmo)P@w4%>)I6-8^AiV<=8>e+(+|6h8?pLImwU-lL7|=}3E!E$riNALF2D1g zIg;aT-Ao%FA5Tm}pNlR|tY*+Dr?5BvvWG+_a&fs}(@!D=_d-<_)8KCPx^JohUL zAw5>*XwcIOQIT++4HT!l4e%C_PT{vEpPzDdyR3H^R%P9^z#N+IQ~|rR?e{fD06{F| z-nRd7<6J3DcbA&ugw%Cl%?Cblvrk>=Zb$UVx>`za2oz>$@aEQcx9xlS-Q&Q z(`Kaf&6BkYLplk3LVXnXBY<1-pU+>>K!=1^~TBgiqWO%3h>*inV86( z?aavPIXa@h>VUyos4O!EPr>Px9f#8NDspwiSeVWY{6 zs;>3e88=6RTwSqug7(SJ?KR}++Jaekg~??j_JD2;W45fTt7E_S{>i=ACqRA2%`9_QUk`*? zjN8GnWe+TJo6N~4Z#}fNeW%K|UOGUa&@STav z@&$xva`G@j_Y-O%5rglC6EeZ~QVUA}0}4CLD=**{d?v2x8TdX!OVyh;qPp`s3)M5r zQM}D!$j(G(RY3dPC+Oeu@hblOS&lg>YHkr&HxM790IgHvKXc{)eBtkjdnOrqoSHh` zWaGG9Q(97@XKPys6!O-pOja~Cr6{ol(A0z+RUYI+7uhEl9EL-qNDvQ(7Qw+MDo%D; zLaWzHb~jpx)44s8gUxe}O@>8w&p0rd{P6|b_`DY;t$|p5^`r!?pUJ6@DZN{TdqTRp z`bHiwoR(+#Ii!x1M-fc<)paI(R|jJ=3p!o88IJ&3R%inR)sO0Oml;*)42*caRg*dL z*PhjepH#qp&e2YA@NZ^e(d<#GlBU2ojy-*LP819%Js80&&YPBS`1fmSGT8?8lU@)t zs)VdH48(UPXB{ZW@tbiH-=-VNYYjF%?NPI_u~DfW9-0}Dymm1QFjNj58EMdnertIU zuvC*BYZTct>$cPZ6eeRc8(&X5`hPb+*-TJMPEKY8swirLxrJ6c40mJ>Q3RirMOqjJ zS}~D3`<8_;6Rg#Gypy+ibkerj9b=u3FVW9hBcTj!k+T>$IzX}0CiL)Se)L@$M%69jP zYiSE=(*Yqae%^oST!k$7Gzxilvk)d)Ewt@k?R?Wg106nNv$|~(e~bjy!pYfrnDKNw z-)D7l80TxGt6L1jf*fzlacO?v&&?9MEDSWRei^PgRo*sxEz>&?miZ@I z;$K`Bx|(SDMEiKpgVFz~fXNYE5R20?*am2B9k~iWYL-KQjJnKtOdncIg{owdi#QL9 zP&?ch72q!Mv`cJJXHQgI*p*(>^_OCp^6sqY?(H2&tD_z2AEnwtI!};9v`eo=4JNnG z_J8O~@&d}%#h=~$;^qK~?6pTylt z=-)p3IhH}XG>^@jTqQF*%z5HuI-5;YR4Mo@xpmN$;YdO4fwmZ(J_ORd3?8$x+y8xe z3WBTPSM6qXY}>Co{Dos2+(KDsf6saWG<8wcLqkL5F^tY59dRP41+VT|uNUwNHeU8!5TINL#66WtG&;xvRWPYA9n`XY7cD86bJhfI8i)&k6)KZoQP` z1+3j#qe*HiwYw*ev6uy?0qp(h+S{+6%)xWq&3mUL2Z1uF&=NRYPUvBVm}S##Ed5JW z)hMJDRXCIfVG3~ff7Bs>YXfkCWF2r61gPjjX`DUUTd2Fg=$#VsvQ>u#=14eA*ir>!C9^rzeidfi=J zYu@FSR2*D%l}0tTK$%FpS99uBC_P`K((oPs!y7)h0X(k=0wr5I$p+Hl)yL<0jv&$G9kkMlb^*z7eUIyxQ#9jl} zqO5iu6Ngqj+&Y1o2_d@?HDMv4Z>ukXzJi2=1O=(jH*dL^Cmcf-!Lybz^rDJWkffmC z!a|MIa`klC*qb8Eg(U#{ijD1m|H*dW4e0km#*a5f?CrfaJ>0bIev0~QzrXw@q;!|C zu3c22^@PFz2Z&I}#w@X3-FgT>yI8B}=>G9ah@fULG?aG#u5#jJ`+Rxu(`sa^c<{Fu zn0jNJgI-U~0JAc`ggXPzJ{EuSzmD{SGi%EW7IyBB;= z>5B;40|JMdp9Bqylt%-K$em0ED~X++hl{9~ZhZGyoP`Eo9l| zXXgLC0EzEd`H;M@?paa|RHV)Pz8(M^ISG{Ja0jmBy2cnR%Sz6+1Wm8RmQ&-Fdh2JR z4kme;_vH@+1zA<ld7GpV+hwRLqR-2IhWPgo+_nX*OI@zi*r5VcuYz)E+#{g{G;{jR-%j%U>6HzGh- z0JB1zsPz;yF6k&=Bk|j`0#4h<`AvwiiSh8rG|vP)IW>zZSif)((((}!mY7-(%%E^r zI6JsF=mQo?!{#{!Lyu3@fMWqr>3EBl4Z@N*FgSCzL2!N6nE!~4JKL5@5teO*1HT7| zX)&VUIZ(=DF~Q^U-F@AOXk647%ASwzOe*f|F_2FZ@|Hl`0w$`SN=@DUL)id8oeT}T zY)@}OtUF?Ine*&ML_5jV@!`qI$pbTE_~4*aclRczAhQAp1X2lm0?1Tg>1pX{!}o9e zfI9;b&nYQsA_EhZFu<;t!q!z1<#Rp*0uVU1@?Y^iNkOr;%AOP0%>d0IfR76)t6H1? z@rPI@P$$dY3(yo1^A#s6D*wfc8fa1Nm@Q)Qo;H+z?Z0yZP3@Ej=?la&;02?E=P$(% zAy_hjvA52-_a=pod~~h)7OGvW?R?QoO$hO#7PvVdm9DC483{>v*UC=!%1-Gk!=QSr zJGus)QhNcI4+A!FY1Xd$zAX>xrLtBpokSSA)2}4NlpVj+#(IT&Thq(Xzh%j(-+U z^y&#&5a+vgw$A-^{EL~#&J%}9!tc_EPZEn>fAv$yZXD7Y=r8{rCSz@9#}4#L7e^b{ zE3^PT;?jJeSWr}?6lVf005E){8VBX1BF*~9%&DqVq$cAm{HI4q#79ATw10cW?43MQ zYN7bg2W24H+#(M1;c%q0M87dJ2Ce~;`8UJb`SQK$E4rTF=FVi#lzr(Tjz1GmavtBT z3OX8PoD0g!ob6*-SX?Nrt5Z+bJJL0SC_^caBu$X>Kpp?2&O;2jPIoa|_)~3e!!L0f zdt4hXmuNi(r;$k09ca?zbQA%#Cpb$C`8@keGI86q=-y=;KPPn#5Bu4xpgt;vYe8K& zJXTk%R}D2dUbnJ?D}u7_NCKn;rMp|hzkH$pbfApLnM9cy7BPz$oSk! zOSGClMo`o0d2`e|h0@B>cv(qJ#I-O5v)dQpmTmt|&hB$)^1^tHm1_3fvA92txs)P# zAzFq^Z^vok@Q$8AvqhrUKwW*-%S)91eoE+_9Q&t&Ds%3b9SK9DO7SS(Bw04sVpVN1 zibPRJaoG|DX>BR8T0T)%%H`24#h=gpB!6>viHrX%v1|@%2&~8f`#e{G8otfjYi z?k?a}`TECkAC<-OmBqYX)MYx5g}U=x?6M?w;itCp+GqJXp%Kn|x7n%eV|c*u*%R!r zmODpub!R8hl8a=P0}CgNRXtCv zorfQR%WIIO@{|*zN(yu-6P`|Mo^#Nqm92*u%VH4_hMDbl)x6gEr|aW@=k%tA7f?V>gqVWOU7x{mYJky0ChYHPj>GNeWQQv zN=0eeG>qNzzJppi@A2QKEBu%GHs;8H@!;R5jx>@WNMN&%nPZWb%2DFe#v%B~H<5<&{ZMB8_6y50w>@!s^IdeG1J0XA@ zY`Gfv#@R80qn`%3SAhM%276cLJrH_Ql`JKPBCr598Y1~n8=xYo7(^h(<@e7w6bWWT${ z;ZCgscF~)kNlVqLc{h0-7%QfT8_D>dqGaKTkUqENpy2U9JpPppqC+EXnxO4!B!0kk z>0&T{|Ct-9_a!m!qi$b^tx{lGnN1@bGdG&FcP3t084&Chm&Kn{2n@7z0{c@>ZPwDp zZ93`MNG6uPTqX@fUY9ddPsND)^7YeThKIawHe^kkJl$Bb-@|P1UCf|{XbMQY>AMH` zvTMPolg<&yE*82-aS{LBx8{;&0n3x(hDP@xlRC=G-ac1> zj^sg)Opn}yFp?=b#5HGOSAt}5J*{#HZenYm7m)um>2Dy3-KC!4Ra~AX6!w_+J1P&R zD~r)VCb9QiJI}*W-F|Hs4quvUA3ho$96ACgYe(Z4Uwz!1$x@(-001M$=j zS~I9V?_unwQzrnBxp9J?n*mdv#gahaITV+@H>&;@Ah%=h-bkdtWMZ@oCakb80fSzD z{@&-Yx7q7T;rH=)v6Kn!l6C37mrM7`06f&Fm&AX%7}E?PUdeLU@Vc{* zp;IrJ0Qe-4g^ZUW>Gv^kwZ#18(n#~h7-O21ye&rGV~MYFWu=@bu#_(lYNex9Mr7W? z(y`+HE*G2ErewOyqB6a$7Q>$UxTJ>Meh<3iZUua#4(3#p-?x_ai|6Z03&1URAx)6> zZkNzc;Yed=zccR&zlE`nC+{OWlbqQu^13A@v4D!l}B7iNC! zR_&4D-!+RBDY{xx#z6QdQV6!nvT06l_k7$c>M<|wEt@_!q2%5x8-F1p4hBhnmsE*h zjVu5=0N@6a3aI`-CkO+7SCLymOD_oQ!{ob{nYtHt?N3<^N?sq?5;$i7@HlL!Ob1XH z$dykPvt6SqnB=TkEPRKN*mvRdkQy?kerhh*t9L@pTUETwc{5vHSb7Y3%0wOjr=@5oVlQi2BJ_WjJflhH<9-OV4Pe| zS1kpzishBt?4u(bDHBQki0^YR=25?uBx8@q5JlUm7a8VODCjf+*OZ|m3A0XH)Ee^a zsJ1DV`9-}f&}UC6DdLEZEd%gw_j;K%t&1B_Rup(E^Dq(2f~jfbF<~?zi$`=xJow(07>|L?tIEGps76T4l+3OVs= z7`n&+U>=)vSV>C3?1EVmvG-aTZc4T*m~~Kh0{0N$y&ikVTE#nDoYzEO?*4D@5&Lhj zL4e!-Sva5WZ7dzl@;#!!wu73Qg{^`+-qgx3FK$8Ge>QG_P+D)=j<)Sf6CNrc+J7hE z_d6tzpu2SN?c?yzaNX4E2&5s=mrwuZf%cciHU0`3D>m9BnrVqT&O>>r~|3N2bId09o%EzGS|R2 zIWW_OfkbKz$X;0dZBe68T_&Pe#oL=_7nv)TWK%?YI17<6OR**z8s1LOIN267kLeP< z{#k(NOKQl!Uwo?ok?%U4SO^phG93a1dt8@N?i4z1-oS+nT_+L~y!#)?d6wM_j&oV= zSwA@&8&7niHhJiBQv@-_1pFX-9Dj4(r9SDZgpU1S4o1r(fTT!taF?YBGxOQ@L}OFZ z@g!|V+)hf`j;2uXW~xS#28r4owegZn4IaOH%02JYUMIa9wfu7bUHa1y8`W{}*{N3k zs9_>2-$>Y=Be#BZLGzBX*aW&p7QF=lf+aE<72k47yMV+;ZrpP2mKwC+-YV@8bSb(@ zG*5jYEVQf_&1{!P#OGQaaMd85lb1TFq#W&+9T`J8wcB>pkby>;kK6FzX!c)3g<=s7!AI5;})d?VMihM4~P=oQT1_w!PPDMA~l z<&cEQgV314!9*$B&RF`n4QNQB+(m7Gf-i+h&#!V^$r5;OAqV6b&Dc$MQ(lYkqS)i z^Qex6jP}rMtta^-BR~&Q)DChC++$G}baMD|W}QG7&XNXiuhyVBC`d|x1?lPO1^xpA=t+Qa|}BZEogy$>7*};(jYSe#WkN?H&x; zx3`>X5WdaZXjPrjB;q`|y}SF^;2Y54Du%&e^9r9fL;_idf1~+Q?*3Tcvm?#dhnZI# zt+!efX=Dy2Y08v5kGfyd(sJ4~ar=Wr_t$^2_fxUgYTdR)7HvPHt5}}8Y}}Iak3Qd| zju=$@T5DHDkvD^QudrtAmy zoisbD;%dCte>(}${if0*fWyz-9`*=GLy-~cOSgFYc|i#(-Ul|E&; z<$mq&XAr$*GiIv^FMqn`RE<=eA=~;zMel(4j3-s!-m{eV^sE8KEtSfxyWqOxvGkI) z?sJMj7?jr68|>`tq)CH;!8*+|-5NV|pPv&j)*<3DS*-Qs!&Vh=(=A%&sN835vZS)I za=vPB8oXYtC#EU2@osl@Oz;;tUDS1?FH@3LrS%}awSDTwPnGgv@GL=+~47ICD9j>7t9k}#~J32e7SLz!z2h`>J_4f}B<^pjI+*x>V za$4INABt6`a5yqE>R2%@F(Rw^4^hWx{cYm(24i zTavI^|1I0|)nf|PJ?NA)rFTBCfKCo~x_+{_ul~l;I}X2;pcQrbzyjRy%X*~4|xhV)Xh_lfrB zK!qOJ5A)z&wEEalm0--Tva?f9Ol3;RZsRO= zLDLKP860X|apxq3ctLDXc(^cYKeK0jYb={=d1fZYp_?V;bo-4*Z)WD{WT$-W2xL8) zOEpa!v=n^yv|y6@etsa#S<5x(ZqmeuAV>Gqpp(U(bu&lqTS`>j2I-@#i*JSyc9-R|5w@@h}U(*}8b8OfWG%u`l7e>Sr& znFH0o&DbK6pM4_0k)nDSW72fEnY%5^nYjG`O}g}r+9b&0coO$$`1EN_Qy95vAikr` zyyQbyvY;ApN_s6>yA#N2fee|)z6YWJ052>6`l+Pi+ShKs|E)kn9m2AI5>#PQ&x07Z z_X8oEV$GTZyan=UT>Ne}?yO*ofI{fb%JBGjsZNmwa9XhuPIREY??p1&K=jRNJ+_(J53rvvVjF5qGv+a%>>&@}Q{6;5XQ7FjEZl z3{Wzv5TJ?fn6MffOy!5)*dT3Wp7l@(PdKn`vQgznB{(b3{uqr>v1Wo2p4i$1Rn7&r zL~^7nzj|`-tcQ7kOGZIQfa2_~dV9QMbTUKeOC|W+4>%c_^~O2~`3vWRuL&+A^v8+?pm<0Fw4TSfTShY=k-rA1T}ErouCK z92yRQck>X_l$Kd#2Rjae-15O&D`5LLrtcQFarPpH`g4*FTvS}(PGgT$(j|$g6-W~ziElwiHueizv1^uh+eRwvTo;YH*(`B;3{o8WJI{>*^7J?++Yq7hZ z>K+HR0*O=SiR=684aV<9eIkeD&hqehlfS3F%?=9)oh98R;BB|Eyi8L?+z+H%pQP?( zp8J2)8buXSl}vbQK3O%EMxfkBWq(sXZWrQEvZh^ctBIn!y5F6Dv2cjT1=)% z^jfK7IABA}3n#k||IKn9{sjl-^vm^Lb9!0)SE7-sW`{fJ&DHcuTa{Et5T3JK1I8UA z{8L(5p66_ZJ?XiAL#S{T-EUVyGQ!>QCZ8vVIRE#T9@IY0`V&JGxk zp(?NedH_drq(N?!Ryl@BH#DfNuSMYxIVY{M8rQT_f=+IkLju|cGq*ZU_e}XZ^FA)v zJ0zLgqKRvcvNu#On;eQc2aLYMo0{~x}3OXVH(5A1WlXRcQ^y9|P%F2qc@0uxLtcDF| z6|w$4$&fm!yF{mmMFkj9rbhc#8k?9TIjihYk6^K7g@ua1v*Xn^sE`XBM}#V-oh;WqD0%Z`GOk%|kD zcBMQAaiU^jzkefH0*oH%d2Bjc<@ zaz}jgW1?sjJc4;+SZ=J=B(aZF!lSXEnoFkLP%UyiwyCh1gwNxEl%B06;K>Bqjc@sf zJ83dl$oBW+@4;}>%GJ{oD=0->wJWN`EpWI$f0JvvC_kro-d&}T{bc5hN+0`%OQcl$ z*wfUl&g9nOw=>fEV2ilxv2^{-zB9n=WM8NB89{BllfnGga4Bo5>KP@{g5mRyT`;4@ z(SGZh-r{B`IiaePLAA1YtQqXvt;d=!;{0N*d21V^S!7B8?(rQ0RDs7#`m^FBj)y9A zM}T~YHXIW?9t95YA}&e;6O;6V-hzG%CTWxIoNpt&S3G30S#X?X{0p=pHV8GxpxrPV$iXbZcQRF*zSeQJ!BJqzp})(cx|zau+Dd?Y)3{5)!F z>ciV#!cKarMozDM6E*^x1$rm-WAC2{{Zo#6VaT$)1RM&Wv%gG)b<7UOGi^2vqMJZj z&TdR9g`AD)Yd0C{b4*>-4GuiS>*u>liA*xkoHhoZ)r}%v23_+;UHpJlR!#=4HSe=N1i+_1~krqS;t)X|TK>U#|e28%qECn4eM zb)Fb?(U^nd-gMC-(l-EfjKC~l!(|3#b#;K)avn!`7Ul21NCl!;`e?N#;Pe2ebWI%y z_j77AQejdAyx=_DWCY$_rFLHhbjtR*%}GlDFyDkfDy**;MK~0}V7z6oILwTi-;LD( z;}{RGJ#4P?KNRFB-uvh=txQNC)Y?C?Z1WRXQTQmjDvF1*A*wO$b#$YUrRy@4XYL zA+$&r5I8&j_5I%W-1)vabLQUbFykb7@?<}Iuf5i9{Z`ok#)P}<;RI9Ogr{HVEt@LO%Ysbadtji>O=mr03< zsR6R#}|}&%^k%7eEU+6{iCK;W53@YZ6THKT&oLjuyBx zdDRWt;~*uUSBCDg=OrVF17(YSl&SB?u`phurV&-4X4v4jpH2P+Kipo@rYSRSxhlIQ zi6|&wJv@2x^Cm#uU_qmC#*FK5?}vtDU@@4|A==~ZF3xZ`yDZL?RaaBH$7~SGe$y&t zW2=pB44+6eeROoR9Lqjal0bk`Vos85!yYh7tN}Sj>je=t9cEnn6FFF%Xl&4}yVFla zr!6d`tak@&=bm`P#4Y~->E?s4IL`-`q|xHdW;y+0xkvV6*2HzrLWI7AD4rBC7d z)lN(AU-s<}t0~C_T~cP%DjLSQy5Q%m(Y-0axRz-a=_O1bGij`pI?O$mO^dUFu1oO7 z0_X4-0hrsTbvg1tUql+_T&suZkt4YD{}9wn#@O|;;R)-!7p{&Q)-b`vm8moSXsnru zDIhUAxXr4g?*m7-5@$FFEC4A2RuTLGDZU`KKmKyE%siamb~2j#`*UD#z}}TuP3Elo z@p|-=c4k9ELrB{LytFwS6i27|7=(-M#~&RX?RN9@g}@axG-AN@X?h*J(?H*qJ1xn{ zZbdo;mO8zRaiyAVo%_=J9Dp%^&=CNqff{Lc?Ty0oMxOtWPylPYn~4k>)(hrhf2At% zsRd_(gxzOnwd@~rW3aI%Ibd~UV`_xj(>Eg ziis?w3IR!-Z16f3qG4@$lm5*bJ5eSBc6Em@NMx7o>FKdNz{mL$soqDHI2F_}CLbN& zl?8OlAw~ApIBspEoqlQb^pEO=JZ-l+G9S+8?$WVJZ?|ksq`F2cxuBqj)9A!EJ2WG0+`eNMtl9%FzB!pMkx6n5zDb#sP5wEysC=}jje+f>n~@kM$36#%dl zeXXJ+_>M;zBn&?d&E>Br8JgM4-mDj=z34u3TWhi{g19VeNGfbnBE)iylP!gs{?|EQ z}!lUo_jq;-g#( zMbJdp#mvDK{g!i(mOu2RFP@o*s%bUUlvXCHQycHR-Y4o7txJRoE8>{@5)kF}KQud& zQOuHmqTTk&IJ26ctiRcj_62}ceAoYomGbx4;aZZi`^W@rESQ({_=jG@cjlr>=Iu z`6IX6cCl6)e8`0N7Z-p2lAF1qud+u|=F&kd16`d-wR9^AkrAVw1W2Z0bjL^7YW#Cet4cC@G~f;WKN-TVECv@ZnrjMX3iYAgsu z`8Cs}KapKMqq9@3sO_2akS6K(f9{3*8pFSl?CIzK+i6+sb^8ZtNSb*NweA))hjApC ztpF5Ea1X@}RFEXsUwQHc0P$#!^(Ky|D8*!-cG=hPzT|!<27c=yKRguon-Jp9U*5yk zqwmQIQ;W2gEy7go?rQmHYLv`iOSZPhUKL;bLfjVBcOQQXz+og>%Z1MyQ{z1 zEsmAadC{?q0vFFIZQH$ryD=mPat`;}>EE)s{!f?j&kEu0@}HAZy)%P*69V~=Y70H9 zyZ6zMrb1n81c7?KqEX$}?p6jqQa=B$!Bd3@!xx%!XOEb&1tdPhwnL(@4*+46{_L-8 zHp#19Rj15n;vvrERbMtE`TYP=f8r)rpC?T9uX=P6m)##?P{_NFqh0^&$sI=2dOQWd z-5}EM|L%D3;y=gxAB(vEhqrB#UNicU4u`|Hf!-p^!@qRh!U8a-Jj!4H2Cf`buF9u8 zh|$8N5FISlGDA32@y##7ntMVI?Jmm2&k2S(?+e_Ec+b|=DdTZ9_MTMR%UMr>e}n?; zsK{j;Ivwy7$o&4Q-XI}6VTfz}&vN>e*)UT(e)CiSBVGKfsE9vX$itUWmt4N*7Owhc z6whdL_`Hk2n&D>Z^1o*4;@vRu!(mM<7gS0`E$8d?-}tKZU+SiSMF2R6*(_ZMRoaw$T z#$o=({vWwyBHj}R3LV$rt+z${X>@eZG$HuriB;7(UZEYnm$OrmCS0Mn9oFYh%+Kh4%BTz$4MZ|KGqZg zKb7u^$XMXAZ%F#*K8^*AUOM--B@coMN@#MH%OtV6uc z!v3^(aX$>XVDe?8nq;HOI9v9&3`MUn{IkgtQ5p-=k>b8eVUu1-jv0zIn(!WsN~tZzi+!4zN|YxJ%N}5@A}IB9ghRgFOCtz z4n(JFO*V^6v!@b*!1{`8hjJ z<|`wFK48t#-&ZRj6TKH6xmR6h*g%bCY8f1r;0Z z`XC>Oc2gWPwy6FlP(l6&c=vySdjE(ESg!vGe*BSV+*<$NKfXBstT z<}O+2AfRR0&-{UM)&{Y=07YS8cxOZ#FAPI0rN*+2;(#zcsJsXG;n6v@u&*~|ozyqE zcK=j6cdjHeKHaL-ZT<2RJ7i#6`Z_0}G6bG6EDQCr?;)v!Obk zg?AeJn5sVD+MqMRroEna{M>z8xF7z>1QPujo15vdxK8DNoAs7+Id-KT+hT)3(`jE# z{k${81Yx?%*gFnKuqNJx98F_DD9`K0(44Y&@gacfA2sVF5?5H=_9j+WSwk8yehZbZ z-ZmYjh-dPXr;p?rFID~ShF9Hrwte}#s8Cqu8%FB-or0hAj=Kwkb00EUv%|&gA6m)E zC<=k;9ql+LWA0BaZM;RX6t)0>O>U|DRe2{%9R%+THZq|F_AN7qT6=%xQNhNuTJe%a zJ?4?~?I!2wW8xV$Ce2x`LQnL%VqMQV2le;->+F_;@g;?GF43I)%8b8wH_fNdF{?gp za=VJKr<&IXF}F`ww!7)A+g#hH0)@XUMzj{_Mf(Hg2yv3U{PyEASex5Ks>T!#*cDlz zVsAqyR4tk``)ZI`Lb@u{UF*SYt1}KN3gB1E1NQ(MXU3K=|Kz?cgW8sxH{FR zSI6~smKT9~Gm1`au6#VVZtg?$B&*jzS3v(oz9b9fImt&tJc|~`0bGPnWh9mbRn3bT z+8kjUY|dY&_A#wzh;%RJ7;G3=f zO@M%|w-m7-ZVX3s4F>u-bCL;=$IPC78=Ojep8ta~QO9xxDp13B#^U@feb|uW!UGCn zZ{fYQ=asw52GE^iW9b`^<=FBa4M_JG0;Vw$@$Q~}fib!uBFs_HBkLigS+5%C;zgmj9iy_;`1bn03 zr!PUcPz5~XiDxn5r6pd1Gv5tIUWe%cRQ;_1rCBamn~NQL-K6@vG0;ZOs;+ZbcV?@f znWQV$yQD9YI?v1sLu4UfGfpq7Qv=HbEbPv7;`(`9H?m5F$a1ruL{a3$s#hT{ncOiJ z-6|o#wmcQ&TL~G*XSzKVA$~eMx_DKpj4|DQ(x-Kx_L%3Apdf{VYIx<2)q z-z_in;SRc>q=c)lu%q`!gEQCOA=saCgm-QP4Q>sk6+$|dYYL}du8a!VX}nr6MI|Hz zE&t60aP_){%eLvmm>CycPj%Q07x{RE;e4s%RjW1D?PRH{T_;xTDovs|S^Eimrppmc!) zotgzU3#<|Zz$U^mX^Fb>qR9@=pG z-ql>HM0##E|Qb1EVWzZ#^$omJr*wIR6hc_SCV-5|M z{gh$G6sveK?pd6@TuUxcTWm`?;$EVL5WUnEmA`iyY+)(euBv;OWJsR{%C+NY)UML^ zLKnM<-1ROloPt#8mHy_CiJyi-c3p>@U&k@&lcL~ClNJ`Dn~CQbv})zJFEIc3*J${%^B_4hXW_801!Lc3LTb#=KRxNRNF zrp-OS^#pZ@$w|``q#kwd6U&jI^WvpbxZvP75eB=zxbppqZl#(LS;4i|fnizye0&b~ zYQS!}e_0}vHV1ZkaOK>Ky7S4pOSG7G0+~QDu@&rq#Jv9EnePJ{JbH<$FWRXWvR_Ci zWVpvAB)TQ!#gm}YR+^uk)VZx}GCs#nPYf+#Yh!jlR(h-L>Z`}(s_$CBO<&B` zuWxQ@nw#g`x9o^yuoTEHEGoi9UU7xH6BBu-r`~3~c!G>&*;TL>n9OmQi5ri@C~+J| zlCMJdx%3ax#;V6Mv0o$B@#z~m8v;xl@Y2sWe;^&M0a`su&9W-Zr4_cuXZC0>^Uas^ z$*C!Z7OlQC8VJMvqu!}XTeXMe@|$Gm)5~qVS@X5_znLgF2t;zOWO?^h~_xQBy}yDQ4{YI435u5t1#3AbGl zv{OqInm{B(>FdBMbnV^Rb~{t!_vxS#&t2}HywZ}7%vh$Hu0iw;7b&sPNZdA5Sme$U zimT!sGK?>K4O$l5mt9&~(y4pp*ox-o<;?|wGgVPhIWls@t?L0JBO@_vp?6v?wEGGHqEMVbgmu{1HPtOcEcZdi;J>BG-X4oCx`FuF?TtG+CqSt3QwWO@m-NNA01n63mW$?FD)v0>>x z!_azekA8_i+cmegwr*P|x`cO*UB5>C&6_vCR9TG{JMO1ST?uReG0Iw6@1HNgQilq2 z3}6-3_uMN>ev2&CO&G&bFAWWikOBms<03}xi=lj2nyb1152_I28?5B#vg5bzE^j#+ z-6!K}@6uMjN0wkdqUW3kEr@-^cY@bP@T-DKtmf274dua?Z{)M;G7C`lLCpKptZMnO z9w*CErNnjCUld${=jJEgw8L)QQRcwKa)7ym2KC8(Dkx9`;-sV$?!B8ym)F>6h-+}Q zyI1Vvr3dY{ovGmyYPYqpc-CtqPhb*2nbVKd$3V z&I?nrp%K^i+=~Jf>wU}a4z(+o&KA?{H*~*@zNJbvANPXXMo_Mh&TliG4wM}1�>9 zmk(4_lRGMJ| z<1xVuPhX)(F^|Tw(18hq8CBZG&8@@g0ON_iS4u6FG`45$gN-NL*)MY>B(#hoVW?1A zFUK%uq%0-SVDcuvq-5 z5N>wd>_rYBppg>A(2;`F(NtUj4&u(ePpKT9a)>b%Fb0j`HE- zz6*)Wih3cVI~%roTrAD)zkj>OF^jHTX73l(z)jHkd@n8HbS^a4`{zz>vT>C%zGSR% zZ01z4OF_OUioGKY6WIWKh81`JQ$#kXN>{C$*2~|v2EDS0#DAt|^>sCaB*X$w)x8Le z2yjP;uLJ#FPcu{b`UG?74YDWjk|bKK7d;eJIs20-Tfx3%H`Ir6SnJgMxpjj?dZQ)H z;DA~}_+GsK>5iAFRjzEBBm@7mx^V!^7Fp8e`Mw26@EhDwo_8smSi39}G7|2L4+`41iJLzdOuWQ2aTT=z zJ>Zb7+S5!2jn1Je9_ukHz6T)6A{ju=lUqDtr&~Sm*Lm^Bt4BDz_JUI#QGIu;WRJsy zkI-T9-e?i?-EQlfG-5H{#2c=%jc&CvBYwkYUBK^57`uBF{n^i6+@zMMznT(s)UFfV zDIfl}5cp%+laR;{l1$YWaf)GwMzC$CAZyB=D+)a_I5S$m*k*${6fm~*K|v%-1ckn@ z7`n9)UnN6duw}Y?vX~OTfBSss^ARH8H0jwIHlqe-u zaHboum20dWXYIQ%+kD4N_Uy+_?&S^PsK+(HGDS8P ziZ_^(x1V9|o82_iRjLD)8ssDhjnUDuPRmA@lC!dPY8w0YetPepxhf_^jVg^8iD8S~L?nt)UwnokGu_pqtJidh0H{IG4=@MYdbNR9 z&-k@UrC0Y8RCQUIb2oaq^tG3RgToV#)~;6|uZ&(oKCaxV1TNvWGY`vWXEh=&f~MqygGplf*R!tYT&H#$ zp3ogH@Vmv==mCefQg>ejf-T9<^`qP5r~Fs>zP@Z5y`nPe+xYX36A)L3Gr5tA3Te*a z5A(W}6VTJIk2=ILM_uw>bNN5hs5k`pYzM1Ty4GSJXqmWu>iY0&36=sOZdlR2bKLPc z)5l97+wXlQ^Z^yXO28|2i)nj@;pQ#u>_@20-#HyM{8oOp(oPRNW|H=heSiOpvRe&1 z&rgOhsI6zJ$AUvTUDdt+UT?NSzE&&p2s5^K8_ z*#}WX5Z~RZD8G-u4BLArdtY_m+0&kVR|lb%Wj`Yw$RI$>ZC3S0?U$t0-R^-u@Dk@0 z5MXSyv$ya2IUN=jwm9~3gjwks0TT~6v*+c`au4({8HSIBebF$MOt$pxr6+6eUW+Bw zi_wnwFIYCr7M>l}E*-6*!p;uq3|YrIrv?qR+PKd;*C#!pg&phdyr4T6&PWkb z@h{dglc~>(ale4PvwL;w=+rboG?f{o9aa=?m)ogDk7;)YVW<^QjiV6*o$^+xVtq|X zMHLkl3le^K#1BpmP)gn@wh0wa+Ij0xUX^p<45^%^$dBHi8k`@JXJHA5^G1*!Fz80R z_m63-xQ2#LdUl(nc#^Ums1&E~qRCgMRwbf@nXAoWq0R+6A}$P8vOZN@pRlKw(z3Ne zX)RE~ZNo}FmRX)e0ZKx`Vk^Y!5V0GVElz(OPF7gQ0h~^J`TX$sxZtY$7o&$1qrAMl zCi-yWiBePeW;J>k;43hJDrj2oS#YU|#r@FaiSdaEA*&zL6B7yidOD>hvfCvfOG)hK z_cxga2Q6fV-K$~{vND9>kKD38O+%lfBu84cqdltL`fHJ&w8RzTIz-Tklcz7(HfWtA ziQ`RHkff;|{k%81FFXXGIlriAbU05Z;)S9YhJ`z#$Cl!5;L{(jAC zo^NVuYGi0AbD=G~03aBEvN}-EW_wsz+&7Ob7*kNofLhq(Y%nIW!~a1H1p^ZG=EXCR zP_B+~-j&mmcN?j7nJE8Su12i_|m1s%Lv!FN$FeP(iBe>b|LWA~WptJ+wIn&zOx@dI<(y>>+ zT%x1iszo$5rZw;=H)&4K54@fnK2XTV_xC1{~!e#;icGFf1g0A z206?2{@f;D2X)<~_u5l3b3)^)5gwG&le?oUrH(n55X~813H`EODc>fz=!M6Zc7UbWl;15z}fD6Fy|_OI{Af#F@0z?H#a13zeuhJ zXc!n21<=ARzLkilak|?8qzn3G19V-gkYH7S!@~x3&vC8`@Xx`VQBqRk#0xA|SlD&D zqjTCnY7irCul8`pZ5=7I!Ow;G+K=VfU&*IHyGGdoZ~V+R6BKa(mb2rb`dmb|7DXa= zhw_rp^kSV!DKvb>FdJJ>IWpJ3^26$BOh!Q5D-CUfsxec`buWrrW0GloIUr-(63F)L zfltn(KzMt56))iVshm@mkqM}@o#J1u5&Z=>)Y8fUVT1TD3>-Ru&mNO_-(mQoo$ZsJ z$+X5R!WfSblN;)WYR@Y|NI4BY9iV&+FA>oWgY?T0;L%P_>HvWQ$XP<@P&O!FALsNR zO@q{AD=>Ay4+cMyotlbku+@p)VSw`FtcgJ76gR#Z(_ok-OxXMcscSp>jOjlt?ypcwbJjT2H zUVbre&C=Pr__rRdMEDZ}u^lTQOnMgXvp|@D=}uLs*?b2d z2Zx8IXYjlCy^G&AzW{jW^|d@-)u({V-Q6KlJFpYYh8^yHrP!_iVWtjk{60!>wi4mi zY@>E&t|dP}oD_mx3Cr7z{8}iq2=sza>eZ4v;iryE5-Og*HUoZLxWMRWX-6hB9j)ykR9@(5sUtvoohzh9A2^Yq`$bz7Ug zh!jOTBvjI1>Q7?#X+keS)8cwUd`hqup31s34`Dkb+0$lrX$yezIa=209Bvp>o#MAE-F?tPiuFpyEKuv-Ad_Z48LXJ=DO4GqAq0<2_qGGcamI+-nC8Z6&O zn4WRu@Wh6^g_Tt#v7s~_!Nm$tD0Z`WVt6=u|GAMU@xq?FXcYlQDX|N-$FOxwz-qs@ zk_aXA!j24k+$M67l)v%Zf} zYifCWY#n-nUG+@c zo8u1XlRO|pr-Ac?Ak;v3^z{o{Tb}|c4oM09w)S9FZT^G}Xdd8+5A(NO*c%SN#FwY1 z^RTV(f>v!XTzQS3a0XtXi!|^$)ei+UUmGJAjYNw3uBC`7)=?(oGvY$U7NMQ<96tj1 zZ!TyT>FA$X%78(2RUR82&O)bpD1x4*la;o*P9)|Tvgfx4SL#u>&h>xNF2g0Rst7GD&R^E?Z>=yZ}g(WywWD&t7*+O)~PbcK3sXjJD*U&S6^e>^y?a>dILbH?0eO{ zHs-p9UEo>*3!Do!z@mY0^?u6GMp?DZ1UGQza&vObpETBGWZZdaxeOdVpjqC5&5L&q zhpH#eda}U62z9KkH|+cv>oz;PL$9TD@=6&b32a*L5__S9$Z%BSMt2`$BBg(M`e>nL z3o_hASO4~lPW*nVbS!GMZJ=C>=jfLw^O@hRyW^f}r3&~TWv#7$gWCu&~_#fcT& zEV+mgiz^^5wBJbv-b#MQ5;u}{yAW6qkoVo=#^W1&3s|JyxetTAkF}qpIjX+QZl1Vy zKC(#AtvR|C=Xz!Zy9#rX6sB;!%F-{FQ&V%4Y)5bAEXCn?k8>=^EpVGS&#a2EH=$=O zTbNVemSaZ!KQ)Wy_MGtxqwLPpR@4FXi02bwyP9tDj2BDOyveVM939GRMM^Hcs}Jpa zyL0HNQ(+a;aM)un7er)yjqAMJ+n<4@$yWjgZ>ow?%ekrXI1cfYu)Mbx=pAeNkQwJH zy*pU+-t4FT7FC$DYVkhD`PNsqiT;iPH@(n$rmG_6LuWm^HF`dd85<%t7?HDEbZWSv zr0j??%(yjxXQI_xiDhjuZE($ot~Kfv<@oSt5xq`RO6DW5R7uGTAoZe!NVCgGc#CAS z)0Es?N<#V=VLIrDFcK)k2PpFvS-ZwwP$lPG&huc3qn_cBpCor#Q-&-9d3CwV}>GSgRw+&4RU6i~6imPhH12L7>E33)2KTj=ZW@b95V8HNeZ*$Kq zSAtdTAQ846{Um!#8Tf;Ni%Lu-@$QJ(N**{@7CM+4lh$mmAI2wnb~aIaAc$N$p54Kw z+1)$jpqdJKh`f2Hl27k7iG$I_%NR^2_HSaUkAj_y}i;L|`>N51u_Ba58{W~L8|t|ODBuPC2Eg}ffF zM30H>^Q6ffH50NQxx&;XCM8IJe5!BHKvj3}*u7kB-0Z^G)AcC|H>yQ?|1wrIK<-W$ zdK71Nuoviao37|e^z;sj6r_OQ0Dx?!+!WH813_~YB&6EmNi65(a)v0Wsj1!kKWGG^ z1_o3>O`9lLJ*bk6?l4&wGUrZ+26ek2n$a3c1??3otS;gUXHRV;A|#ZJ)IZVoY&70_ zHEkDVf8iytJ^)*p_L0nhM07yCg!>7QxfS^D}jXR-y z3db_%6%-WAKMk7adw>7f`TD!ZxYB->7?#?LkVG1-r~MB8LkP}`l~q|J*VZ`CcP=5& zw=c}VmKHnA9ZV0mQbmJXHPiudye9l8E zl9oF;I{PT{Fr+9>1rIp|OzB*T*BTrf6Tq+uto;f$;nsHjVpT#~Wl-#<3Jsf=He@3$ zsPerQ(9B?uY&9%GsPd(~aiKNX+8C_@trsVp*ZMs#5-st)vapfrW8c`!&Y$fLk-Ie8 zy{6t8J!QEPY1$f%%1Kbc#AKATl~)m7&=OosGZdosK$qwJ9C*#0nZ1l8ubF=wKzbwh zoZzL~0%|jpo1^SneEZWA8_|_s-*`4Xell_{!LT6VYKuES+__f9nuD@-`8(`c1SO^-3@H^o3QkVGZh2e`Uvwbn4k>YU!rVlj?wR+D zQF2gUw>B~|>b%jOj|a}m0nLA1ljc=b^P*=`tK#`M!=7eNg6fN`L%S_Kn30hrK0f}! za`GsJUCDF#wBu>?gPsD$-j~oGlv-YNeYMrRryIE0G=+W3L3oStWTQ_bNap9oxI}ei z&1k|vp3K6m8=^#>8tGCw^Q{dK)io~G9jimLv$L}1eA;O)g4DEHUoLb@_mn*TlI#S} z+Eq984pwn1{LaQn-5E#V7t2X4GF+lAy_c{&WrS~Zky`U7Y*fJ*T>*)^!C32II zim}DNK51^#sIijuH5ny_RIDV0YC4>poa`S0ro?u?S`>_Zz(U=-u`$2JrP?;Io2mbZLM*{k^ObT#x9>DNX-yWe{9={Xx|89u18(F79* zBGW7}a+YVOC!n-cjg52&6yPYk7FpM$K}!nYYy+3ZGUXas;FHCJ;|*W7sE+Ysc3Dtw zGM?4PN*Ow7ec5^%sEWWR#?!VRRBQ=mN=4P__6 zqIQ11zfRjV^)heq`#>E$l|17945iar>^_FhU}znEiAIcX&r2!xTFKvBfb8vF%LXXp zDi({BN}sr_1d7_L@GT3=z`=g-byC9HL74Ml`_}}qXeh6(_0FGm++j)Q7q0C6@ zjmZi$f+T5dZ3K8%>tm%lCLnjZc28QAR-&LxD6ykHf)&v|Zn&7&V1I_e+!Wg$9={%M z1zxP{S`s!y6s%U~9DC{h(r;L5j%ibo%LF3VNo7wKgqO~BLG9>3>cR{u%ZYfa4t@BU zb_0F=tE_aQ`r=*eavDuH*#*g$Pn`z8ELztOlaOZXLg01kc49I)uP!30*6gdK$W z85NV6jy=KFb0z2ITD$^6A(ggCIL|2~BO@}y9|6cq;5L<=O`vzcxv@;N;YwB;qPIVF zW(O9p9UKGal^OWQVD~YjYe2=O@LH@jD4zl80!Z}4Xtj%`Zk=k(lZa>EXb(WW-Fll2 z`mH#i9`-5m;|o^LkTee;ePN^77c{rj7vr{KIUTLuc*;q#{ai29SqvgcVg6L!OF}pw zO|%HJsj9&S^H5B%za2I3v?Hr|P+9jJX_X-k*T(Q`hAyffO|$y-s{cGqfkpy7hOVn3 zd0d^oq-Qf(p10U1A}qX4Y6Z^t&<4FWMn_A}0rFvJ#tl^n%&PR=o4~S)5JZ}!iSZ_# z>juTYP=AG%+Gm$8UR?IPtWK0F@nLAEcsYB|%j1W|qc`>;GiUP4(*{2ChmY>ayi6wB z>rEJxkkHO%FS}o##OyWotT);+-=yW#A;D0Cv8ztpciGA8TRDIj$oU?10evndnl5YZ z5N3D1!4!d-HhqDg%R}1rTl;m^#^5^Zkvh@4)tD^_Lv=nwp%H z>%HV9j<wJPtO4{N$>A8G#arcqZ@uU;svp6y`Xn(bTPYayR4IeO0G!5G6yymcvN+ z71XC&*%Ra^X$U_#2e62_0&?@MWce_bqw*F36=1^jLH4P8W<^k}Tu%*PWWFLChc`7IM@}I^5noO!_W1{P_N@61V1< zq0!N7P<+?dxfWOO4M-laZJj3E`;Tq!vbN_vNX9W_g4C5$-P*xMtT(Qn$18K?4y&?CGiq4+W5l{Z=;8u;u?)FTNmD`*Ey>~eN}Y<^fR%^x&8IX3LY zBq?c#N1?$Az?TZ^8eRoxv64{)(Xp>I`fwKTnCh%4weTV?`3UD+r(b>+LAW4sEc zU3zg$2jb^K-1g?l_Iyr?$F#i8@+L2m=z0W}tsbM02&ZGDv-_r;-IeN%GCwv>Itg@6 z;Wc(U1*_Y<`kPK7mzK$k#|k9r>A~i85Y#KX5IK@mJKQT<&{jRRRtWbTJ-z}^MzK@D zEye_SD^MNDK-8V0pxL(rw8GAQ;@;|xe(_4a^_YeW3RP2L+DYor((q$yy&nAoCAy%V{8h-NXQJZ4D0Ff*C;0Rfq4<{L?%Swxh zL8dfniX&jyAmeDYrYq%0g)K*ZKy-F3mP1#ORnVwXuXfSYYq`jJabJBrZ=60z_l+j2 zXUp?zOY@`Y0oQwX+s&S$k5g*4zaA`?Vm{?H{(9jOaES;ZfAT`qXM52yEv;w z{*FK)DJ&*GsF%!odL(2bQZc7*O|+Jn^@cFY{4HLne^veM zG8wNOCDECVoH=>7x|ZpASYjHjWCAqyAGRD0**_vO3VZHR0!3DaUyulO&E0^p>3{G4 zAyoO670FYd5F@fuks6(UzI1XJoG(+RBd=56DIy{=0@9-z8pIJpmZ0f?`DlJB2!&g0 zmbk{CXkFJzNkVCi$s@(bmxtHv$w1*#9%)=ga4h9^A156i&^7+9$XhZyl!{4-WAptC zT?Fj84|Dda2wdt=h_P@onAx}C$Hf_4u_D7?SxA})YGCiwkma?zbFOX*=-xtaDIXWD zL#z2fJ?W%`*`D~L?l%dt=n4&|2iCn7)+S(5_X9r>NF6;HXK((Xr)armYqzvBl438c zYx~Pw&XMnf%VyTDfd|sr(pgisn&l84C;!0O;0te(c?w?7TMQp%Z=^ed(RLy^Pe3Wr zO5e#fN?2{Mv0k%3?G&n$(^qol!4=-sU(*HcgBcV$d{~NfcM%uCb1+~Bveow0P9J3y z5UdnnDc6SLtMHr~Klu4Bn2<)aZIrqzIH+(|)Y59{C{5zI zT!}6+q1`VBp%-@cxZU{gSL+X@y(g8&M4+97f0%9PDCs)6TaJa7>Qwgk_h8*R9=5W) zrdHTy?a$vWnIr`5nvT2W@@m{y)nj;DR+*b}T?tzJVl>VfLj5G0#M`b&x+`q+g;twP zeBHiXMRuKJxW?q@c3njB?g(ATN(Y&<=~P)tVQ?IU>D0}7htHlJyQS~+5a*X`%qBI& zV>}-n71ZB@dp^wZ5O8-HVabW*gtafVv%1Bu>BmXcm`seL_dAm=oD2vI9T_F5`M^A& z6Zxv$90fh!zR9J{`;Fux3G0}dwQssD0b0p+Db z)J7WmbVGIy5bWmb?rzvwcG<})$Axm&TNy&UYen0tPj?1qeflsNIc1b^^Y!490ues1gMO9Ys2nS(Z@|cA;ypWCBpV-d1Vz1J>=^BbU} z(QK|Sdre2m1)&e3NLanHIz;XtU{Rd1AXbt4b9(eK53r*dDngp}F2!Rac3hi#@Qt7> zRhAhu=CPx@F>(`M)Wq0=xXLIo$#iN84v!Cn;@7Zr@z!!FJ8I#oISO9KhZq{OWD1%j zRk31Z>~WkG$Sh+b9t13k8EnjjJ3;+!#0Md@OME>a)LBfGu9`>gCCh6r$D(QmGP^Y4 zRNh+E%otefuyxy2x$IVWgh%pVC&Tp%TI4#SeRqbUXJ6H(b9$RUxy0z1Mkj2yI-%uU z<6Z%UE>q~xJmh`#?Gt=!?P1n;xqunk-4Uex$*+D7B5~KwVM7W2kM<|SKz(FuX;o4> z2@w#Q%QJ+L@&^8)PRnRLMRHdJUds)e-M2t-f-$*UV}2g(W#+JPLo9m#&vm(EL^v=;!Dcc@8*ITHp`F z>p(xG$0XMt67~#M&c7SmfiG-)t?GY}M0d?AEcqeeSNeyF7k3w-hmq`p3bKZW-kNd1 z8ZwXx!dm2VNRJ4_j;*{q>DymC8`EDN&PTi?!=LrcNk5xT8n6h&PTkH?0Q!ta$E_M< zz^)B$sbyc4ZoI{vIr!z}h59YmN35h2HR6Kxazjtc?Pi2GrjSa<`x|NxqEtYTQ88IC z1FYF|Hq8;|ELG->>1ws#p-W}p6e;skR_OA|e_AeQd7E!pGIwwcI?6o&eFRw&=~EPjdY+u&dkL-jQk`98ej4BfGs-8X6I;dx4*s#j$!@q=4)uf+#9 zRhtYYux>R}{-09DLxG<^W&0L{`5E8sYLevanLnmaE~r-K3!nV_IDOtHHSG7PRo6kR z3AfA%^7$a$2{e1zdZh>S0U!+l%0}<4%dL3`kp^1=x(Iv*OiwD()y)mC?)v+p)ga#+ zOhcZ!-0=YxIfg&lw1?5mL!)z`8pvBEJBJlPR>Ad`WP#u z?_44k#;vvLeF``5Ii0y#@8aajDi_P~aCeI+`jj=6B8E0RX7*fpx^zu?m~P1ZC|MCu zvv2}(@l`>gr+oK(;J1glgLZocCRheWxg;pJcX5mF2^-W znm+?UfPkmLcxf@g)JNld*IQ7XYlTtS{yf<(@BZ56&z>P~26MRVz73n0Q`0mEuVez?T7prZp5Wpe9nkKOJ3)R(Q{YV-*g_T|YlK{>vNmodGma!Q5rP$$RG?OX;jQue^SDok1-M z`IZrAQb1?y2SW`J*VtTE)XiM+=>f}Kpwu%oc!;p~0el)`mkn=&LrKO37rkEEz$fc8 zthc95p4T>jEQ`I7V(*dDrCJbFrwk)+I*gQ0Ai+;JW+fBZEzqhRKCl-$4iS(Vdxi@cwMmKD6V?DePuk4$iVFo+wF_WpqXJm3MH!9MSE$}z-u3F%2y{9 z!B!)$<<}a$`lbG8E7M;SIf}mU)SAncqqhCT`**s91+Nu$yz=!apw(ZA8gP#rf^_(r znws^X}rlgE$)rn;xG+i8zli2SJ;@V8-4;Xx_3raPvDaa>dY!I2K`d4j;_)U48v&^bBlw z@wDP~&smTh4z2uVkY3GVgv z#cf_Ick^kmxm@m|nWf zg8$=;Qu9;LF%X0iae9X|m^nqUZ{<)DvZ_=>e=@Iph^;aw(2fuEU` zOe)A40l5JkQ=dNj+%4cP|C!b$Cc{+_gtl>etGghSbU=IcUFSu9HFibzbrYF;&sTV0 z=uo>8C~uBbn*zJOk=0h+5kXnby%au|-NkWt$CcGj(yThQE_R^nu9WvM^i$&jW9Q=Vm( z$Mn3PbhSkGGM_7AlM|8`>$s!GoVV(Vh+-S$+`1@pu6=r&_c%qF$%KwSUJmhK1|>{iQl;^s;!RDW z?9=BS#Cn)BOjE$M6@x9o1Jpm4}o@fI))+46_ z7X(tI4JNu;lr7d?T^AQvjJxchwXn^}jyH9o*(|};da0Uv(6o!tYeg=n5(Y1~`Lb?M zvg4kWq$T3A#~?1Y3)7U4h#nK|+cn_P`T+QIj>1y-dp?% z`W{8d_@oY%yimi~JYwIAH`BUf_+eJubTx%18|3XT(2>(YWFv?ApWhm8demRt%@9mi z3-o-gep$t}8T8HsI}fWq0l;8sI6Rw%FcRyXq_GDc@BeD=J%FOxx;Ei9dqL%T6%Y_G zfCvaEIfJ4gAVG3Q$r;Hx=>@zZAW0-NjglmbjJX|Y9eeDDbxCSY{-u}^v}VD<$ztOp+gCMq*oe8-Q@i;lfNX~G zpAGo}G-B$mH`OeI!Qxc6x1UeN=yJHcX`f8haC!;GbW&21?6z}m;J`m;Smfrc3Ou8g zM1P7v=zsR_xr*t$$YWnumX@B2+`SIwDML6yTxH7gXOB;gL@f8vlgI3#mRYMRTRLE_ zMq!C4Z1+#?lD;ZZI+f424ieqlYjNh26yp7F>MkFt7UgXllT!hsPM+&kU_jf=Yajjl z?q_8UN{ry^r}Z{^lw_9eq=mO!Z<-xSX9^zSyz0^)-FeZr%*qkPBr7cz+pS5|n~3}p z?t`2sf`Wo`LLI~+qk>o9Jc_TD;}qg|(>pE2!0w4k8t~MEbOWrxS#(d6Nyi-3g9831 z3N~$;M6sBJd|m6m!2(;fDgVBzYlV(b%%BXFrVMY8&%2SXlW^k1iRi6Fe`Dd{s!B;LOip?r-Bfs}!1G)`KNR?JxbY8>9z8~%0c zS$WP*zThFsLFY7iiM2b5DN#-r#uw9TcB1YNZD%u->!(paW~ZV9U?z#!ZaB6?A%SIc zqdIdVwiBOW=Q<5~yU4^~eG`2zVPlh}u$fxU@Tz_d9`6#{z5Dmu!op7R=MY{U+Vd|r z5?ElvMIh9u5ZWA|K44t%3aQ$FOfn)1=4@kF1<`V8Ge?p^m5DNxx?MPR;sjj8X)v$e zS=g5AYVOd}o>!Zjo0R|DJ{#V^cm4WxOBFn9)?f}K-vw4HSe+naWOHym!cnsymQPaw z8wyf@QQHPNINXM4^RiyJbP8f_UOfwTmBr+tr}rhWt}@2rhA-3-vf>A|^J)nFZ|EgS zTHpAVn&^f$dVPe(7bm#5@B$hSRIc7t3@%!z=s56aJ+ce+Eg6p=o*C-s5yC{=HR(8- z1q^p0YM@+FzykKUkqflikgS8f1`%4Mk^<5PsRjNGSxbEgC0Wn$FC_4Mvk_rDR04;I zf_0uqYzI2MW{vBJHGP(>PhAoGQ(zx#hh*~%Tgvx!U$&U{y;r`3M-7t)=P27$#B55k z`u#jzD*l(_RGS-C>CfFRc8_W`v(BiNi`y%nt3^83FqEn~qJ|ZVw>WPaxjOHgvatCP zgj~~R4nU-IY}ftGJ_%f>?T_>bf+Spc%`is&_2)a~kk{Q=$h^Nj+%}Hu(-y_yCzkD8 ziYto<<|)r=G?cg6s@_xb2YczMJPMSW=5(9t?JX~Menuso;z6AEOYB}yP0Nq0KzGcK ze2?31>OJx9)X@=;TD2~2eii9|2V9g1j+&+2S#^*KFx`v%b7|(a)uinu}n?kV*~HaaSUEJ%b_5{p>i!pRuX*5OA(D_Pfq1F)5`vE z711%a1ztkP^Z~e*clnrr)X}3yk%a}@UpzO^8thVtk!JvsI9l%lgq`**!cZEgCpah9 z@a&Z>Oy4_`B&m)i_5dp(GnAkvL;-weXk(K+%Rh5w9sOn<;Twv2YjuVI`2!^=mU1<1 z`+QQN2bb1FG6d}lm5NAFmmJey;AeJ{d9oEuOh*fZYS$IEXNn&i2u5oz{++q>mNkC>f5%r! z)9|}mSsSZN3@H_&Z3G`0xckTzMxF7l@fJGsiS=NQD!UtL_tp$z}IA z^`h9HO*4pYFY_|Hsdyp)K+Vkza(1#;qJH$>K5^=lTx4~K1AR*a9 zwi8I1)z#ZoL6RDDrQhFWX+a$(Gywqo2KXBCzF6=k`JI6tiV~pY6;|%9JiA zaUTlsANnir6<=?T^9v7XihU{K)zVj|hi$zrwrqtT*|X&`2h9o`k3}7OauWb~tWizR zH`hO}G@n6AiXjIWIYSCWp_Ut9$c2SbR*cCfz}^wfklMi5J50CuTo#UD^}$S9d&p|r z5}jLA)A{sn1V6kw$WOJB7@}&a>bH5j#0cdaG4h%NJWhSir0l zdR&Tlikxu{PRl<+#TK;FJH}1b$@8FUCtmC1y%y3R#ad&Gv~KXc8yEgWfTc3@0yn$kSy-dR>d(pR$QFSvb-u06OnjU zVa#Pl&6u&-b;M`2+w-{Ud$z%8zO|p8)hqRdaSzZ1eFFp8Cft^|0%%(Fxb8vI9F&5g zLW6wNEXbOoGlwK-y9-8}$v=GtJ-ReNBj)qKlOrTO<$>IY&&B-B8W9yCRHQSLIvy({ zn9BmmI)fid*PBuq|IUbCLEt#6rr4neR^s8W2YM>TNNP1NEbdm!ac8{js>71* zh)aQUuBaVK1VxbRV(CC{$aKWNaKTR6bE@xEa@lVzK#ZRHLwAl>>6wHd9`tCVr0}?h z*q%$7a;kw!8W(t?oL0cyLwbE6!7=}50DJ+1vgT(X7xcpj=Q|jG&Pf;biu~>2Tc5lj zKZWtRn&RA}Cd^*0KM_q$<+V}hxp*hIBF~XnA&;iSk!5M{vxxaz$Y88})r%KVB#=pyn!_ZM2)_-@p3&UGjIYxPz8y?dMqeCGK}RV&602KT=dnaWCw_p5n*QnTFo zr`FA5;gq~8YHCxia<=T}D{RUOUuW|lXywo}nC#MJQAjK7*2m+X%K;uBe_69jM$*1% zTw!~uyDAKmCI0G>NLn^$obaHGrH1b4cYZ8lh2)<*tW7D&qmTXpDSZIAL-i}f8@~K- z*lbOO9657TlLOt5zvuhiM+VinF}WlskG0!3S%x`$d_G=J<=T-?H!TbrWA-V5;{f*P1v`OBVF-H( z9C8R?sSwegsvJMfnt`BTRstjFV5Dm-dVGV>!Ba&l-5qLfg&=u<+mk#C?J=9sCE#ip}$vzIm^Y z;+#CEVit23*geD}xYkLfO2B0$$#W6brW~3d@s2~sQa`9j*`F7fMRPSM#nHAU=+szD zWrx})UY(>5;*`f%#OaY29Zi&23?aJr(}U{IURH9$dDR$QS9i8&ci zvy{fg3g;jYG{gBz;5Dlqv9Fyl4j0q7;SLeTA5?=q-yH!u2;nEr@^f6hv~L8x*-Brb z?R5XZ;8=iLS&c0%ARyowX7}^z!LG*-pC6GRui{xvlu|`N@=~D3StDY5mlO>5ump%EDR9$HgD1We@R2A?VW1~$SU@qP`DdgLiZ^XnhvM~Jt< zg9O59LJ>~t;M5QT^=PJ$=1t^hK~KASe(u7`6h5QptWG$Mnz&S|I^|xgT4PyMc%FuD z`3K-72={EUK{Rdh+gwUqC`XYrx)Fj^JgU*CjrrKIV=O}&b_QN~;5^W8&463qF-vsr z_Z@;Fgto8KcKt=ND(kCY5zMq%0D_pOq_2sr7Xf>ecm<@nM^P&)osiNw^JubP-fYfq zxvPQ(^=J;!+8%`GnYC*>PZiDUx$adnNKtb|MY5`f5NImq6)~LpW|N0vg_k6E-x`SN zDErd($MmI7{ncKjkL$jbIk`a)7y=Nf1%cB5bgCDVfr$Ca{7624 z?2)trR6(pLHbCT&tLPbpnV2U0?5Z>m&5yq3PC}{`QV8&6xCHpF4b|x*^QAcc_w{B+ zRR6jC#^$)jR)xmq)1gYY0gNhucv+x9)Sel13wZcoSp#4a{4Vg7@b^F+t?oX$J+C2{ zZp~yqWshl`3UPCveiL&V)%9eZBS;+JbO2~kFAWlb7<0M1B4fqJv_QhbrZW@Eoeb>* zV~%og*iJnb&P|n|F1!(&_-wyQ3kiRRd?|7wC_>*MuRNja*OQ%vIgo_rr)b`qTwU-@ z^mA1eOlynNUkMtnu6u*)1&TVB38>JMoHG24Kuy3p1c7ZV#@6OQ_|gwspw(*7`}eKa z+O{`=0`GbPf-YLAewlC+!Hi;mMdK5Egx?I*Fu|?yidczB6_v`*CwMb#g?t zZ&h;VWz`Wb9=7c*WdroQ`LjbPs#w*js*cPC!`4u1dA%m$>CD}jMDeP6W(;BL@HO0* zY!Tzm?~ZpF|0w6Pr;SkzH7S{hRQ)6H)~{4V^j=h&rHxO(eprwDYyrQrc6d}4c|hud z)1V4z8L*t-?U_mw$^fqkmX<1HEcbI1EV)YHo6N{D1Ji-@XUe~L6L~19@ax(DNO5dIFVFH5HoH{?v z!{%#KPI{a}xYq0X1|YA#Tumm9&8r$A~V7ZqC$iQ{&@q)A|ZPP^6{T$8># z>$mPhQ@?Iq`}5ZP!@(!N-iKV50DtJ6Qw{s=-KTyR)!6#&HL1fI=4TEmoaYrTJFV_zehOANp*SJgitvdk44IX9`tR=N0 zGIvgUong=vbMmpCBJq>8Ui`={6hg$Q_5`BF4G+(4R40g4h-)b?-ZmhK2IWD76Mj@ zgfe~u2s!R_42RGBt19;CU2LJ_~4Dhko3G%K9$qKnL{aQ753LOsZ{bO=9k zQ;D&0dgu>xTiT$r6^)tf8cDk+WrHqq=suX-NLFp{C8KQ%WlR`g9s1MRDj+tF8wRe- z0d74L*smK#!Fe_AX4DyW8DOEG%+4!Z7iB^dvEE)iuYr+C1FgZEqH}_C#xj$Yg*c8 z9)V@PTyP_^dmbM;?*wObHmbs7vXp;4+G~Og})Bq{*Fyu^v0Lhi9w2TkTq_; z1M>fxsV?6)a1hlwzI+ZbT4aJ10QzFAL(s)uk?MGV#J%j@@ z`E`z-&D&`Rzp%hZ9@)LRkMI}ny?8I)y723tDE}+}#_Ngy z`xn4&ISDewDoS|4k}FX^r4W*)txrjWF~?#3Bf&f~8` zOG9&?KOuMdI{x$VfJcfi_2233uI0*rT)S@9U9;#r(qb+4yyC|r!KJYeou14Kp1HvX zAB54$G2CyE5!ZlAqD@;k@lxU5G7RAvJyJKHcnSz8k?WaTiJjnmM}^7nk$NjHF(UEy zvOPog@^w_}eft`r)M*iMz3GR^|InmcD)zsLZ8HPlxb=;s3e#r%Kj%Xbb~u!z?N(oD z$Hqr{r_;dk$14$jAAgI5GUtqkD1#6hYbh4APi&6rDt`j_OYh4WEUQj85FOF|tE*=+ zR;*ltMb>jpGOzwJEMdvZzd4GrV!JIrSvIT`&84rnxF9^ia=%tfa+sLkg$(0QM>a8- zWM+i6H|y7Glce|Zy+vMznaSAyHB|AzyC_LC0xDS7YzP)V7CzCSeiUwm8vo6zoRK7< zym2CVUdHkdxQqY4I<;jai6~br4}7MA&=yLP_1Eg`dJQ+eW5_|ja-NRU;-~VC=-y@t z4f@40EUcDC*0}g%yXbG&7?P&{`WDNo=Ba=rE>&h^L|VT##9JO1&+Lqu-2D1#h8WM@mz;rA8-EA6rezqy!?AI6kDxt6s`P4NX7U~l~9SfJy46y zd-Sr39=?C|%XNj$#I;Yb#=XT2Ll(L?RD%@9M2i`Y?p+@Kr_g46wg@kT>;(GohRYdF zo^+1|&+LsE`AG7A>ekQGq1eB#+Da$*X`>GmVWs^ztWs;T6NL3n9b!&(dUFLaH4Wifpfn_s8^4cF!-39OJVIg*zlU#aIqE=*n1@ z-}4C>*^}SxGO*R|g85PnWWwV2Sdod(lKH(`Do5pF&j&g8k(q-Ufz@-==vBIqB^_uW zb%0fTai4n0D#hZMq(iZHr|ZWco7kb0e7hV-9^`sQLo+YpOF7^z(+0ApW zlb2=ZdO{l;l+7B6y0k>UJ|m=*8;qzFWzko&ZZmDpKc)>{|$<=d*= ztLKy}zBsCDEp`Is)YZaOn-hDgS{*sa+Lh1kf9QW_fk~oU!sMhfA1kJ{!5!gyCqKhU zzW06qX7Sx`G}XSmT)`K`mf`}784#v(+QYxqfl$VcmEEBG1IWRVT%i1`840>_)~b~1 z{0mM)V$QNMyoHzim&i-?$2&r^^{pIK$>4nOpZcxQ35kkQB-VqldOq+vT-o12@J-48 zuZGiD)-qfMxss{UmX{)w5ly|1QM9edO#8hX{ELmPv2fKY{9G*pT`CktSDUOmD*o}zRI2puo8~)mV z{cl=({~9#P-)uEv*R=wymQ2QN!a6J*RI9AoeiIVplK;}JQ+hQQ&~gd1ZQY(s@OulU zq9=Pg3(StirKi3dDh9E6*PY!;|0msevgSTsnGJGRR$_mlEW|&3`UI(7VehqfNj@vu zxjMyi)cVd5J$d@skO9-7pg>{&_|>+8=KpLw7@{IB4B4=1CsXnC%&Jm$AmvLqHXyQ|X#Z|JOs$py1$T zXtj2imV5w}C=vGGl{GfDuI{GKthHejFV)>>GJhtd#q>rZ2U9RItdgKnJduRi z_3q<&RI~PlYYJ}NeNtdB8N!v;5PNbRC{n@Q5Ead--D;`rlGyqr@!ZQ+w*TG3C*r&7 zI3+U6qsNZ95YTRNkqq8H_b8+`zEjr)-{;Yv&8HY~Xfs7+wR zCcG_6JkmrWn8{~*+OQX7wk-_%LD+HZDU;>=a5Y@F2Z%Jaz>un5HP~E0b_T4R*(o&A zjveHbx3<>DFDUpHQu5}5=QMM5Of_j{hblFZrTnw*SVqqKgV%$20)8!(R-tZHEEvVm`n= zQ}SZ5(@r(Z@||EezH&u3f1tHW+b z@2w?mc#Bg=Bm(7?pyAG9EDsR-nsDNIMO5GF0RU*~8b7P234Wna311Eqy*QZ?ZU%V`kC}o7s^Ji;;wkSdS_e+=ShJ3kU+G%8fli?1KXK*&1<9~|Eg&kI z2qltCLO*)EKE(Ya2{VeHND|b+G5Y#)V|={RyHilschd@^duy^1t@E!Eg7dEtrds66 zP4qLnjf=asdd!>RN?mpT+1^~`@~-X@5(@Ft$nJg*1x(EOx>jBl>*~U?ie$G!ID9!W zl30trs)Wza&bIyh`6#lytb2wb+Z2M0vJ>A2x2|Qxhf6j!%}c!FHAjvG$pr8fg(Hk{ z*mOO);q8eejqpQ7<<^F(&3Gke0;BEZ-NYZCZzq zojR|zR?)Y93lJ%ChK7cK+;|xg5s_Xud2a%D8QL1Ca)`@Jj=wT2h)!kBwwcg8z~{w9 z-H`X?CWTh~zR7ju-Fh3ErYeoc-rZKF*<9-<595>JN2aW+Tb*;>r5a5HY4gQnethWI zD_qad7rCpqO-@>LetVa=Zre*W;=~GY8zI*jE#2UdFY|W_ImR_UeYf3!b78>wOIKCw znXYfQ{t`4SMWJXCWN+V8WBT>LX{)&{XGRj6x?|mnJkKVd)Gw)tHuNxLmhjzf7N4qo zUmA@W=!((Jn{Hf@NNA24$;$8ed)9bCTbMv=GNIXWt%l%8)ezKR z>5ueAOm7~(@Xa~jz_9ty?$n#AjmTKP9vp?CHr9Tq51m`zT}{^U+MSk=MMhUlqp?g#^(7#2Vlg&0Hm85&I`cIGcmkPa89XOn z2hQVUo)(tlyl-2%rYmK07^E%9HDrC6mj|S&kZtF&#GJt?SPfvW#1Ne-|9LIHZ!R(BZXZn9&jv> z7g$1>fcc4=pkSIIU-2Fo0xurNM(*qPxH|f}bQ3 zn-JwhMni(CwD|V+_Va>|E?mBR*#l^&EB?Gkg()cXAWiVp$7t#p8jgIFi|1)8clc31 z=+Ho)A!Y|^tFPA24;UujL$mpr!EzqZaT(~#y!`wnV@|-~zMi@Yc(>i6Ra2;&LhkGc zVhj9}x0cNk=42Gp^6lF<8p2^kmOa~3l8!AlvQD>hk(5y3=-|d06PCI}#(^)lP39>* z>73qv*K8^@@s+G}jR9`k??%i*rqSnQk-4(VaMN<2Iiu3ZNTlV1)yKoj>s*iGoewz1 z&o>aTrbLl3fzE?MqRxSV&G#jC*&5m8>>BX1l87$h9!{vX!Zg#G@I;oI zTJ|NG3Tai?Wo12gDsn7*XEK~B$=t+*soQEfXF|`$#Kh#`1n}WTl{`z3FtjZ1 z2e`%3>MERTeoch<+7dVfA;Hfc@09Dl*u+9APQi^$D9l?dU$sqBiC0i+aL#V95Q&jv zhU_N0=kw#vFZXnk&Uj(C5+TG8gDuu{9V6`Acgt=Fc)7X8sARH4jME#`Z|0yACY=#* zHT@Q>vGQ80BkAQA^oLBvE zWc=^6^Z9KPqg)E($e(_8k;S|OWZZ4&v_g$4A1QM?KY0SX>m089YD^?@O6RSrEe`mJ z_f0vVv>xvrXts7gXJe{%d*muKY6d9vUW9p~}|;&l0)CtjXJ6TE%#R$j}@ zXC4rtfrFP(3wElTY1k z5+#{9{>83RbJA8w=oX%G96rA8wF5t#-yXK z?_^aG(LQApDb26f9kDzeKeD+_P3N)URY}hp zW7a&xnY%j!!CbO#QM(-y1_^ERK4xq(;k_iDQjNucP<^7sVkRj?sjF}`pGB3ES$B7# zqG~-p&E9~{-ekm3X~KTTQ0$OISA^kiTu4XzSm8!SjVjJ>z4ITYrek)wV~0vP#aarD z3)iD+zPfWr0qWMpR+^d+Y@=bn>O33;ca8E=F&Ri zw)&6tj4Sxa*J26vwChn^_r}>}rq&~@h2Jk9qUNh!HHM$Yq|OkpvPe{BMh=W<2xV&{ zM?{mM)lNa1K#fR(#MYoml3ziDfSx)wpOdHMlVN`c3o_J?J)j+nT9~(OPt?IRmE2+Ec!KUTQkQ zV6)gttdcRVnfy%X;Mzu*_gY2|U$rkQiEzvZc4GOP(nMVW=Ls2u-iTPYQc|{>obMEe z^{R12HYdmUvth@Y<*k0P@XSV}Q~0M~ic-%L$#;6@?5Ppe`Jt_qck%mY8^wD9D)F*c zGH8i*@1Ko-R)Y;{b0wWNE=Pb{Bq9Cv#FsOLG!pH1Gowv+gLa+8JQi+q-{@|XXQ7SL zo`8k>wehpbWmaZY9!uetdtx-3b<>GSXsWzLy|OZ~x)K+M)^+K* zMjML03m;$f7P91}W#ub(_qLiFj_P!p-*ha*g#IY*6D57LITc&!sP8tdd1Se^qADG? zrPHh4d{Q$+5;c4G*D`*eE3HnrTdd(eg@G=Eubv$ucnx+|5zcUNtQS&L z7e*R-o0B+}`iFif^^&q{{fwG0ej5Vu_M6A3vsf_R+=e0>_w0t3!ZgA=G}zMmmYnaE zdOj~v%~b1XFL+Oxv)5qw18r$X_@E{`s%^;2v@>UHDyAf`n@p06Fq}>4AK6VV+0;xh z$VElx%!t*NRYvO$e}%)Sbvu8^rFi#g3)@F)=qtGEiR7M={~TVaFV7uD)=x;KW8|f-e|A;g4TOIO@EUu;>d~F=|5@jK?Z|Pe% zQ&y)9`Z_U}*5fzsV0?_ONUXmsUcW!&#KpJRN*<~a?tq=E*_`XLDz+Icy8*R3V%ZId=_}+s ziI0?EJa_Ic5cwi{1uFeGJ%+$)LQHW`$BfB7#{p#O%fnSZg#fO(A#uYI{8qz zQU-rP$z^xeZ!Q^Qz~xk-o&P2Iru4Oe?n*W`6u zflrC$MgIrCLAOhoE&QGG zx#nUAY`aZL#K;b*36TYi;MQRPn>Bs?ss)z5o53Y~6@&)}ml@aZg5B;2jg`zh(-mmW zofFI`KY*?8UklyKwH$FX5?)u@HRqP8?cLGM4pm}i91)1)@+%ykx{vMK;(ydM5}XKb zyjpfQY2`fNv=Bccs`X9=IFsm7uS2|-_f7f)K}T=?+LdEx)_X{Wmc3`y5-i3>cI$5M zEH)K0`bUldC;h!ho8c-U@O_9xqHH9I!3t`c-and`3OSQuV`F?JflzcLLCWNU|Qc}PeR^X-%h%DENVuIj<;5<&C z_Q>Ft)4?46yH@F%Q}^D;gTUzIG|hv0@7Fj?~i{7I7Hw$sGi*&NGVBTe~i*dnEi&V z5PxzkI4vsK-I1-iJK8qlTQd+19gTOopWW@aI^@Q+wcbTIq{Z>8XE=4@I*%Xt9Y7F} z9_0(Iy`m9sJ}%@A$g^d59u{VXk3E?8FhZ6i8GyRN^m0s~;`e=Ce{#8E9sQ@0=49O9 z0F^^b*>Mv8dS;+>KIamiYwIJ|HLo^2UKw1&ip78f23LN?#gdW79-W3z6^{WDcY~0( zz`oq|SI%Q#UnMPU!u5#r^SE!{9)J^KxPwP=J=u@t=qW?2wk|Lqz>}`^)>N1yML-;3 zGg4Dc6a$(1_y!#k4BwaFx5qcQxg)_(?ZCwH)ViGiptHZGP}I+>y;eEBs|VLbyRG=3 zdQu@*qXlvFAdsK4;*KK3{JPbj$o|Vpf~V%9gY`gqf_SRH0e||qmQ!;s10rqIx0rW+ zyE}n}zA&q}sT;78WgR#89Z}W+zb}G}mq}Jh33qEy8;i%|KlX-{Mjp~4-ydc2PY5VL z05WPU=a6zFRGg;f=X3pY{BVxgzJ^nF2TxvS%@b(w*ga>qv?XE4;9wJ6IQ_fTtaxcd zRcnR#_+mV(ho5=$Yg|qf6;QQY-pjKkvdDy=$F418kcJ0_tEX_wNlH~kQ@D1l_%_|N zuGqn%wEqXq-(K*hL2EyIhEkT?9}Y_Lz+PYxp8QBx-+nYFc{1^zLz3+OtB>vl#o#+< zU+hceKfK=m(H~Ma&qka>l)r}1VyOLCwqf-fIL4{pUX}?n;f~P=zZtME7rhsGb6!4S zhJ_bLwuZH`We#vEFT2D}a2ygS@jOA6(`CU+o8jwZWyQAaSxreQ%9WP$u#=8BvUllj zHP*KD4C9J!<07 Date: Mon, 28 Nov 2022 12:01:53 +0100 Subject: [PATCH 70/82] resource ids in example --- .../network-dashboard/deploy-cloud-function/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index bd4d0141e4..bf5d5e90d2 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -27,15 +27,15 @@ This is an example of a working configuration, where the discovery root is set a # debug = true # } discovery_config = { - discovery_root = "organizations/436789450919" - monitored_folders = ["321477570496", "821058723541"] + discovery_root = "organizations/1234567890" + monitored_folders = ["3456789012", "7890123456"] monitored_projects = [] custom_quota_file = "../src/custom-quotas.yaml" } grant_discovery_iam_roles = true project_create_config = { billing_account_id = "12345-ABCDEF-12345" - parent_id = "folders/321477570496" + parent_id = "folders/2345678901" } project_id = "my-project" ``` From 314cf62b692923530fad4de4521159ddbfa7464a Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 15:19:57 +0100 Subject: [PATCH 71/82] discovery tool readme --- .../network-dashboard/README.md | 21 ++-- .../network-dashboard/src/README.md | 106 ++++++++++++++++++ .../network-dashboard/src/main.py | 2 +- 3 files changed, 121 insertions(+), 8 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/src/README.md diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 768e0f12d1..90f8c302ad 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -1,4 +1,3 @@ -# Networking Dashboard This repository provides an end-to-end solution to gather some GCP Networking quotas and limits (that cannot be seen in the GCP console today) and display them in a dashboard. The goal is to allow for better visibility of these limits, facilitating capacity planning and avoiding hitting these limits. @@ -14,7 +13,9 @@ Three metric descriptors are created for each monitored resource: usage, limit a ## Usage Clone this repository, then go through the following steps to create resources: + - Create a terraform.tfvars file with the following content: + ```tfvars organization_id = "" billing_account = "" @@ -29,15 +30,16 @@ Clone this repository, then go through the following steps to create resources: cf_version = V1|V2 # Set to V2 to use V2 Cloud Functions environment ``` + - `terraform init` - `terraform apply` Note: Org level viewing permission is required for some metrics such as firewall policies. -Once the resources are deployed, go to the following page to see the dashboard: https://console.cloud.google.com/monitoring/dashboards?project= a dashboard called "quotas-utilization" should be created. +Once the resources are deployed, go to the following page to see the dashboard: = a dashboard called "quotas-utilization" should be created. The Cloud Function runs every 10 minutes by default so you should start getting some data points after a few minutes. -You can use the metric explorer to view the data points for the different custom metrics created: https://console.cloud.google.com/monitoring/metrics-explorer?project=. +You can use the metric explorer to view the data points for the different custom metrics created: =. You can change this frequency by modifying the "schedule_cron" variable in variables.tf. Note that some charts in the dashboard align values over 1h so you might need to wait 1h to see charts on the dashboard views. @@ -45,7 +47,9 @@ Note that some charts in the dashboard align values over 1h so you might need to Once done testing, you can clean up resources by running `terraform destroy`. ## Supported limits and quotas + The Cloud Function currently tracks usage, limit and utilization of: + - active VPC peerings per VPC - VPC peerings per VPC - instances per VPC @@ -55,10 +59,10 @@ The Cloud Function currently tracks usage, limit and utilization of: - internal forwarding rules for internal L7 load balancers per VPC - internal forwarding rules for internal L4 load balancers per VPC peering group - internal forwarding rules for internal L7 load balancers per VPC peering group -- Dynamic routes per VPC -- Dynamic routes per VPC peering group +- Dynamic routes per VPC +- Dynamic routes per VPC peering group - Static routes per project (VPC drill down is available for usage) -- Static routes per VPC peering group +- Static routes per VPC peering group - IP utilization per subnet (% of IP addresses used in a subnet) - VPC firewall rules per project (VPC drill down is available for usage) - Tuples per Firewall Policy @@ -68,6 +72,7 @@ It writes this values to custom metrics in Cloud Monitoring and creates a dashbo Note that metrics are created in the cloud-function/metrics.yaml file. You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`. ## Assumptions and limitations + - The CF assumes that all VPCs in peering groups are within the same organization, except for PSA peerings - The CF will only fetch subnet utilization data from the PSA peerings (not the VMs, ILB or routes usage) - The CF assumes global routing is ON, this impacts dynamic routes usage calculation @@ -75,12 +80,14 @@ Note that metrics are created in the cloud-function/metrics.yaml file. You can a - The CF assumes all networks in peering groups have the same global routing and custom routes sharing configuration ## Next steps and ideas + In a future release, we could support: + - Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) - Dynamic routes calculation for VPCs/PPGs with "global routing" set to OFF - Static routes calculation for projects/PPGs with "custom routes importing/exporting" set to OFF - Calculations for cross Organization peering groups -- Support different scopes (reduced and fine-grained) +- Support different scopes (reduced and fine-grained) If you are interested in this and/or would like to contribute, please contact legranda@google.com. diff --git a/blueprints/cloud-operations/network-dashboard/src/README.md b/blueprints/cloud-operations/network-dashboard/src/README.md new file mode 100644 index 0000000000..47318cd346 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/src/README.md @@ -0,0 +1,106 @@ +# Network Dashboard Discovery Tool + +This tool constitutes the discovery and data gathering side of the Network Dashboard, and can be used in combination with the related [Terraform deployment examples](../), or packaged in different ways including standalone manual use. + +- [Quick Usage Example](#quick-usage-example) +- [High Level Architecture and Plugin Design](#high-level-architecture-and-plugin-design) +- [Debugging and Troubleshooting](#debugging-and-troubleshooting) + +## Quick Usage Example + +The tool behaves like a regular CLI app, with several options documented via the usual short help: + +```text +./main.py --help + +Usage: main.py [OPTIONS] + + CLI entry point. + +Options: + -dr, --discovery-root TEXT Root node for asset discovery, + organizations/nnn or folders/nnn. [required] + -op, --op-project TEXT GCP monitoring project where metrics will be + stored. [required] + -p, --project TEXT GCP project id, can be specified multiple + times. + -f, --folder INTEGER GCP folder id, can be specified multiple + times. + --custom-quota-file FILENAME Custom quota file in yaml format. + --dump-file FILENAME Export JSON representation of resources to + file. + --load-file FILENAME Load JSON resources from file, skips init and + discovery. + --debug-plugin TEXT Run only core and specified timeseries plugin. + --help Show this message and exit. +``` + +In normal use three pieces of information need to be passed in: + +- the monitoring project where metric descriptors and timeseries will be stored +- the discovery root scope (organization or top-level folder, [see here for examples](../deploy-cloud-function/README.md#discovery-configuration)) +- the list of folders and/or projects that contain the resources to be monitored (folders will discover all included projects) + +To account for custom quota which are not yet exposed via API or which are applied to individual networks, a YAML file with quota overrides can be specified via the `--custom-quota-file` option. Refer to the [included sample](./custom-quotas.sample) for details on its format. + +A typical invocation might look like this: + +```bash +./main.py \ + -dr organizations/1234567890 \ + -op my-monitoring-project \ + --folder 1234567890 --folder 987654321 \ + --project my-net-project \ + --custom-quota-file custom-quotas.yaml +``` + +## High Level Architecture and Plugin Design + +The tool is composed of two main processing phases + +- the discovery of resources within a predefined scope using CLoud Asset Inventory and Compute APIs +- the computation of metric timeseries derived from discovered resources + +Once both phases are complete, the tool sends generated timeseries to Cloud Operations together with any missing metric descriptors. + +Every action during those phases is delegated to a series of plugins, which conform to simple interfaces and exchange predefined basic types with the main module. Plugins are registered at runtime, and are split in broad categories depending on the stage where they execute: + +- init plugin functions have the simple task of preparing the required keys in the shared resource datastructure, and are usually tiny functions one for each discovery plugin +- discovery plugin functions do the bulk of the work of discovering resources; they return HTTP Requests or Resource objects to the main module, and receive HTTP Responses +- timeseries plugin read from the shared resource datastructure, and return computed Metric Descriptors and Timeseries objects + +Plugins are registered via simple functions defined in the [plugin package initialization file](./plugins/__init__.py), and leverage [utility functions](./plugins/utils.py) for batching API requests and parsing results. + +The main module cycles through stages, calling stage plugins in succession iterating over their results. + +## Debugging and Troubleshooting + +A few convenience options are provided to simplify development, debugging and troubleshooting: + +- the discovery phase results can be dumped to a JSON file, that can then be used to check actual resource representation, or skip the discovery phase entirely to speed up development of timeseries-related functions +- a single timeseries plugin can be optionally run alone, to focus debugging and decrease the amount of noise from logs and outputs + +This is an example call that stores discovery results to a file: + +```bash +./main.py \ + -dr organizations/1234567890 \ + -op my-monitoring-project \ + --folder 1234567890 --folder 987654321 \ + --project my-net-project \ + --custom-quota-file custom-quotas.yaml \ + --dump-file out.json +``` + +And this is the corresponding call that skips the discovery phase and also runs a single timeseries plugin: + +```bash +./main.py \ + -dr organizations/1234567890 \ + -op my-monitoring-project \ + --folder 1234567890 --folder 987654321 \ + --project my-net-project \ + --custom-quota-file custom-quotas.yaml \ + --load-file out.json \ + --debug-plugin plugins.series-firewall-rules.timeseries +``` diff --git a/blueprints/cloud-operations/network-dashboard/src/main.py b/blueprints/cloud-operations/network-dashboard/src/main.py index 9a2e2b876e..5fea7f664b 100755 --- a/blueprints/cloud-operations/network-dashboard/src/main.py +++ b/blueprints/cloud-operations/network-dashboard/src/main.py @@ -297,4 +297,4 @@ def main(discovery_root, op_project, project=None, folder=None, if __name__ == '__main__': - main_cli(auto_envvar_prefix='NETMON') + main(auto_envvar_prefix='NETMON') From 2fd3e94555bbb517c9c5ea4e6364239176deb7c8 Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 28 Nov 2022 15:35:35 +0100 Subject: [PATCH 72/82] top-level README --- .../network-dashboard/README.md | 165 ++++++++---------- 1 file changed, 71 insertions(+), 94 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 90f8c302ad..c59a70c2d9 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -1,75 +1,18 @@ +# Network Dashboard and Discovery Tool -This repository provides an end-to-end solution to gather some GCP Networking quotas and limits (that cannot be seen in the GCP console today) and display them in a dashboard. -The goal is to allow for better visibility of these limits, facilitating capacity planning and avoiding hitting these limits. +This repository provides an end-to-end solution to gather some GCP Networking quotas and limits and their corresponding usage, store them in Cloud Operations timeseries, and display them in one or more dashboards or wire them to alerts. -Here is an example of dashboard you can get with this solution: +The goal is to allow for better visibility of these limits, some of which cannot be seen in the GCP console today, facilitating capacity planning and being notified when actual usage approaches them. - - -Here you see utilization (usage compared to the limit) for a specific metric (number of instances per VPC) for multiple VPCs and projects. - -Three metric descriptors are created for each monitored resource: usage, limit and utilization. You can follow each of these and create alerting policies if a threshold is reached. - -## Usage - -Clone this repository, then go through the following steps to create resources: - -- Create a terraform.tfvars file with the following content: - - ```tfvars - organization_id = "" - billing_account = "" - monitoring_project_id = "" - # Monitoring project where the dashboard will be created and the solution deployed, a project named "mon-network-dahshboard" will be created if left blank - monitored_projects_list = ["project-1", "project2"] - # Projects to be monitored by the solution - monitored_folders_list = ["folder_id"] - # Folders to be monitored by the solution - prefix = "" - # Monitoring project name prefix, monitoring project name is -network-dashboard, ignored if monitoring_project_id variable is provided - cf_version = V1|V2 - # Set to V2 to use V2 Cloud Functions environment - ``` - -- `terraform init` -- `terraform apply` - -Note: Org level viewing permission is required for some metrics such as firewall policies. - -Once the resources are deployed, go to the following page to see the dashboard: = a dashboard called "quotas-utilization" should be created. - -The Cloud Function runs every 10 minutes by default so you should start getting some data points after a few minutes. -You can use the metric explorer to view the data points for the different custom metrics created: =. -You can change this frequency by modifying the "schedule_cron" variable in variables.tf. +The tool tracks several distinct usage types across a variety of resources: projects, policies, networks, subnetworks, peering groups, etc. For each usage type three distinct metrics are created tracking usage count, limit and utilization ratio. -Note that some charts in the dashboard align values over 1h so you might need to wait 1h to see charts on the dashboard views. +This is an example of a simple dashboard provided with this blueprint, showing utilization for a specific metric (number of instances per VPC) for multiple VPCs and projects: -Once done testing, you can clean up resources by running `terraform destroy`. - -## Supported limits and quotas - -The Cloud Function currently tracks usage, limit and utilization of: - -- active VPC peerings per VPC -- VPC peerings per VPC -- instances per VPC -- instances per VPC peering group -- Subnet IP ranges per VPC peering group -- internal forwarding rules for internal L4 load balancers per VPC -- internal forwarding rules for internal L7 load balancers per VPC -- internal forwarding rules for internal L4 load balancers per VPC peering group -- internal forwarding rules for internal L7 load balancers per VPC peering group -- Dynamic routes per VPC -- Dynamic routes per VPC peering group -- Static routes per project (VPC drill down is available for usage) -- Static routes per VPC peering group -- IP utilization per subnet (% of IP addresses used in a subnet) -- VPC firewall rules per project (VPC drill down is available for usage) -- Tuples per Firewall Policy + -It writes this values to custom metrics in Cloud Monitoring and creates a dashboard to visualize the current utilization of these metrics in Cloud Monitoring. +More complex scenarios are possible by leveraging and combining the 50 different timeseries created by this tool, and connecting them to Cloud Operations dashboards and alerts. -Note that metrics are created in the cloud-function/metrics.yaml file. You can also edit default limits for a specific network in that file. See the example for `vpc_peering_per_network`. +Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/) for a high level overview and an end-to-end deployment example, and to the[discovery tool documentation](./src/) to try it as a standalone program or to package it in alternative ways. ## Assumptions and limitations @@ -79,32 +22,66 @@ Note that metrics are created in the cloud-function/metrics.yaml file. You can a - The CF assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation - The CF assumes all networks in peering groups have the same global routing and custom routes sharing configuration -## Next steps and ideas - -In a future release, we could support: - -- Google managed VPCs that are peered with PSA (such as Cloud SQL or Memorystore) -- Dynamic routes calculation for VPCs/PPGs with "global routing" set to OFF -- Static routes calculation for projects/PPGs with "custom routes importing/exporting" set to OFF -- Calculations for cross Organization peering groups -- Support different scopes (reduced and fine-grained) - -If you are interested in this and/or would like to contribute, please contact legranda@google.com. - - -## Variables - -| name | description | type | required | default | -|---|---|:---:|:---:|:---:| -| [billing_account](variables.tf#L17) | The ID of the billing account to associate this project with. | | ✓ | | -| [monitored_projects_list](variables.tf#L36) | ID of the projects to be monitored (where limits and quotas data will be pulled). | list(string) | ✓ | | -| [organization_id](variables.tf#L46) | The organization id for the associated services. | | ✓ | | -| [prefix](variables.tf#L50) | Prefix used for resource names. | string | ✓ | | -| [cf_version](variables.tf#L21) | Cloud Function version 2nd Gen or 1st Gen. Possible options: 'V1' or 'V2'.Use CFv2 if your Cloud Function timeouts after 9 minutes. By default it is using CFv1. | | | V1 | -| [monitored_folders_list](variables.tf#L30) | ID of the projects to be monitored (where limits and quotas data will be pulled). | list(string) | | [] | -| [monitoring_project_id](variables.tf#L41) | Monitoring project where the dashboard will be created and the solution deployed; a project will be created if set to empty string. | | | | -| [project_monitoring_services](variables.tf#L59) | Service APIs enabled in the monitoring project if it will be created. | | | […] | -| [region](variables.tf#L81) | Region used to deploy the cloud functions and scheduler. | | | europe-west1 | -| [schedule_cron](variables.tf#L86) | Cron format schedule to run the Cloud Function. Default is every 10 minutes. | | | */10 * * * * | - - +## Metrics created + +- `firewall_policy/tuples_available` +- `firewall_policy/tuples_used` +- `firewall_policy/tuples_used_ratio` +- `network/firewall_rules_used` +- `network/forwarding_rules_l4_available` +- `network/forwarding_rules_l4_used` +- `network/forwarding_rules_l4_used_ratio` +- `network/forwarding_rules_l7_available` +- `network/forwarding_rules_l7_used` +- `network/forwarding_rules_l7_used_ratio` +- `network/instances_available` +- `network/instances_used` +- `network/instances_used_ratio` +- `network/peerings_active_available` +- `network/peerings_active_used` +- `network/peerings_active_used_ratio` +- `network/peerings_total_available` +- `network/peerings_total_used` +- `network/peerings_total_used_ratio` +- `network/routes_dynamic_available` +- `network/routes_dynamic_used` +- `network/routes_dynamic_used_ratio` +- `network/routes_static_used` +- `network/subnets_available` +- `network/subnets_used` +- `network/subnets_used_ratio` +- `peering_group/forwarding_rules_l4_available` +- `peering_group/forwarding_rules_l4_used` +- `peering_group/forwarding_rules_l4_used_ratio` +- `peering_group/forwarding_rules_l7_available` +- `peering_group/forwarding_rules_l7_used` +- `peering_group/forwarding_rules_l7_used_ratio` +- `peering_group/instances_available` +- `peering_group/instances_used` +- `peering_group/instances_used_ratio` +- `peering_group/routes_dynamic_available` +- `peering_group/routes_dynamic_used` +- `peering_group/routes_dynamic_used_ratio` +- `peering_group/routes_static_available` +- `peering_group/routes_static_used` +- `peering_group/routes_static_used_ratio` +- `project/firewall_rules_available` +- `project/firewall_rules_used` +- `project/firewall_rules_used_ratio` +- `project/routes_static_available` +- `project/routes_static_used` +- `project/routes_static_used_ratio` +- `subnetwork/addresses_available` +- `subnetwork/addresses_used` +- `subnetwork/addresses_used_ratio` + +## TODO + +These are some of our ideas for additional features: + +- support PSA-peered Google VPCs (Cloud SQL, Memorystore, etc.) +- dynamic routes for VPCs/peering groups with "global routing" turned ogg +- static routes calculation for projects/peering groups with custom routes import/export turned ogg +- cross-organization peering groups + +If you are interested in this and/or would like to contribute, please open an issue in this repository or send us a PR. From d856057fb298a1a0cf8ef29d3d6d46c8071a77d9 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 29 Nov 2022 10:16:11 +0100 Subject: [PATCH 73/82] Some documentation fixes --- .../network-dashboard/README.md | 20 +++++++++---------- .../deploy-cloud-function/README.md | 4 ++-- .../network-dashboard/src/README.md | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index c59a70c2d9..647b885e68 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -1,12 +1,12 @@ # Network Dashboard and Discovery Tool -This repository provides an end-to-end solution to gather some GCP Networking quotas and limits and their corresponding usage, store them in Cloud Operations timeseries, and display them in one or more dashboards or wire them to alerts. +This repository provides an end-to-end solution to gather some GCP networking quotas, limits, and their corresponding usage, store them in Cloud Operations timeseries which can displayed in one or more dashboards or wired to alerts. The goal is to allow for better visibility of these limits, some of which cannot be seen in the GCP console today, facilitating capacity planning and being notified when actual usage approaches them. The tool tracks several distinct usage types across a variety of resources: projects, policies, networks, subnetworks, peering groups, etc. For each usage type three distinct metrics are created tracking usage count, limit and utilization ratio. -This is an example of a simple dashboard provided with this blueprint, showing utilization for a specific metric (number of instances per VPC) for multiple VPCs and projects: +The screenshot below is an example of a simple dashboard provided with this blueprint, showing utilization for a specific metric (number of instances per VPC) for multiple VPCs and projects: @@ -14,14 +14,6 @@ More complex scenarios are possible by leveraging and combining the 50 different Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/) for a high level overview and an end-to-end deployment example, and to the[discovery tool documentation](./src/) to try it as a standalone program or to package it in alternative ways. -## Assumptions and limitations - -- The CF assumes that all VPCs in peering groups are within the same organization, except for PSA peerings -- The CF will only fetch subnet utilization data from the PSA peerings (not the VMs, ILB or routes usage) -- The CF assumes global routing is ON, this impacts dynamic routes usage calculation -- The CF assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation -- The CF assumes all networks in peering groups have the same global routing and custom routes sharing configuration - ## Metrics created - `firewall_policy/tuples_available` @@ -75,6 +67,14 @@ Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/) - `subnetwork/addresses_used` - `subnetwork/addresses_used_ratio` +## Assumptions and limitations + +- The tool assumes all VPCs in peering groups are within the same organization, except for PSA peerings. +- The tool will only fetch subnet utilization data from the PSA peerings (not the VMs, ILB or routes usage). +- The tool assumes global routing is ON, this impacts dynamic routes usage calculation. +- The tool assumes custom routes importing/exporting is ON, this impacts static and dynamic routes usage calculation. +- The tool assumes all networks in peering groups have the same global routing and custom routes sharing configuration. + ## TODO These are some of our ideas for additional features: diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index bf5d5e90d2..6ac87398cd 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -16,13 +16,13 @@ A few configuration values for the function which are relevant to this example c Discovery configuration is done via the `discovery_config` variable, which mimicks the set of options available when running the discovery tool in cli mode. Pay particular care in defining the right top-level scope via the `discovery_root` attribute, as this is the root of the hierarchy used to discover Compute resources and it needs to include the individual folders and projects that needs to be monitored, which are defined via the `monitored_folders` and `monitored_projects` attributes. -As an illustration of the interplay between root scope and monitored resources, in the following schematic diagram of a resource hierarchy, the root scope is set to the top-level red folder and it completely encloses every resource that needs to be monitored. The blue folder and project are set as monitored and define the actual perimeter used to discover resources. Setting the root scope to the blue folder would have resulted in the rightmost project being excluded. +The following schematic diagram of a resource hierarchy illustrates the interplay between root scope and monitored resources. The root scope is set to the top-level red folder and completely encloses every resource that needs to be monitored. The blue folder and project are set as monitored defining the actual perimeter used to discover resources. Note that setting the root scope to the blue folder would have resulted in the rightmost project being excluded. GCP resource diagram This is an example of a working configuration, where the discovery root is set at the org level, but resources used to compute timeseries need to be part of the hierarchy of two specific folders: -```hcl +```tfvars # cloud_function_config = { # debug = true # } diff --git a/blueprints/cloud-operations/network-dashboard/src/README.md b/blueprints/cloud-operations/network-dashboard/src/README.md index 47318cd346..4f9597f2db 100644 --- a/blueprints/cloud-operations/network-dashboard/src/README.md +++ b/blueprints/cloud-operations/network-dashboard/src/README.md @@ -58,16 +58,16 @@ A typical invocation might look like this: The tool is composed of two main processing phases -- the discovery of resources within a predefined scope using CLoud Asset Inventory and Compute APIs +- the discovery of resources within a predefined scope using Cloud Asset Inventory and Compute APIs - the computation of metric timeseries derived from discovered resources Once both phases are complete, the tool sends generated timeseries to Cloud Operations together with any missing metric descriptors. Every action during those phases is delegated to a series of plugins, which conform to simple interfaces and exchange predefined basic types with the main module. Plugins are registered at runtime, and are split in broad categories depending on the stage where they execute: -- init plugin functions have the simple task of preparing the required keys in the shared resource datastructure, and are usually tiny functions one for each discovery plugin -- discovery plugin functions do the bulk of the work of discovering resources; they return HTTP Requests or Resource objects to the main module, and receive HTTP Responses -- timeseries plugin read from the shared resource datastructure, and return computed Metric Descriptors and Timeseries objects +- init plugin functions have the task of preparing the required keys in the shared resource data structure. Usually, init functions are usually small and there's one for each discovery plugin +- discovery plugin functions do the bulk of the work of discovering resources; they return HTTP Requests (e.g. calls to GCP APIs) or Resource objects (extracted from the API responses) to the main module, and receive HTTP Responses +- timeseries plugin read from the shared resource data structure, and return computed Metric Descriptors and Timeseries objects Plugins are registered via simple functions defined in the [plugin package initialization file](./plugins/__init__.py), and leverage [utility functions](./plugins/utils.py) for batching API requests and parsing results. From 50b583cbc9df5ff2785eee0c28286f1bf1a01dff Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 29 Nov 2022 14:47:01 +0100 Subject: [PATCH 74/82] Add secondary ranges --- .../network-dashboard/src/plugins/discover-cai-compute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py index a22b5d644b..4138a88ad1 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py @@ -195,7 +195,8 @@ def _handle_subnetworks(resource, data): 'cidr_range': data['ipCidrRange'], 'network': _self_link(data['network']), 'purpose': data.get('purpose'), - 'region': data['region'].split('/')[-1] + 'region': data['region'].split('/')[-1], + 'secondary_ranges': secondary_ranges } From 19191c171924c24fb5eb1df797d8b948b64f00db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Tue, 6 Dec 2022 13:30:01 +0100 Subject: [PATCH 75/82] Update README.md --- blueprints/cloud-operations/network-dashboard/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/README.md b/blueprints/cloud-operations/network-dashboard/README.md index 647b885e68..1fe0960f7a 100644 --- a/blueprints/cloud-operations/network-dashboard/README.md +++ b/blueprints/cloud-operations/network-dashboard/README.md @@ -10,6 +10,8 @@ The screenshot below is an example of a simple dashboard provided with this blue +One other example is the IP utilization information per subnet, allowing you to monitor the percentage of used IP addresses in your GCP subnets. + More complex scenarios are possible by leveraging and combining the 50 different timeseries created by this tool, and connecting them to Cloud Operations dashboards and alerts. Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/) for a high level overview and an end-to-end deployment example, and to the[discovery tool documentation](./src/) to try it as a standalone program or to package it in alternative ways. @@ -80,8 +82,8 @@ Refer to the [Cloud Function deployment instructions](./deploy-cloud-function/) These are some of our ideas for additional features: - support PSA-peered Google VPCs (Cloud SQL, Memorystore, etc.) -- dynamic routes for VPCs/peering groups with "global routing" turned ogg -- static routes calculation for projects/peering groups with custom routes import/export turned ogg +- dynamic routes for VPCs/peering groups with "global routing" turned off +- static routes calculation for projects/peering groups with custom routes import/export turned off - cross-organization peering groups If you are interested in this and/or would like to contribute, please open an issue in this repository or send us a PR. From 7d0376c58c5282da18678d7f1c38206a91f865ce Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 6 Dec 2022 18:55:18 +0100 Subject: [PATCH 76/82] add legend to scope diagram --- .../deploy-cloud-function/diagram-scope.png | Bin 52705 -> 34406 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram-scope.png b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/diagram-scope.png index 9ccef15892440f45df2c3f8f8ed226dd8edc731a..6247c1c90d7872c5eb9a8525d197ac1689204219 100644 GIT binary patch literal 34406 zcmdSBXINBQvo6}0Frgv}N)`}Q1XK`^Xh4yys32JZ$vH?4iYOTb0m(>4Y*^$>iwX!x zYLev8X_86j2yEgXkgKs?f4>`jhGHXT2o2bl2`biXu86_ogr_JorLMBQ|;ZduH z8RP?Nv7TS<{gBo2>Z<3xe=(q7bbJ@8A5m%2W4Jz$(XH<<9lfGUCumaGwM}f;UcWgs z?id>rV`XAyTC;KIytG=+z}jYd?PAg9kla%WRNh-K1G8t{XrKL(4fhxg^>pB<)%JaQ z27m8GZP@@C*6{lL(c+ADqfe&qw2h60JNWqrrg9?6MW_nR)C*4>M0F$94PptEK0Bsa z=2uL!d_VmmO9FMHdWroRy_D2$>K|oI^~y4fMuwy4zs5G&yNYaak^ZmA;yn?cIe!`R z*C`NMPHs?s#90R3$_lR8gC%G3IgY-SewLc zaOHDU*9=$Zb9(cb3i<0djZbS_1vB)L)?$UPlYKsY037}JUnK>U;>6*nE~E$<8lG?M zT`~uhZTYjpl!mLeGQ84eA6Tyv8s3+SgjhU61&P98a6b~I_1F*g@`$ud-l?bKXv9+K z(`at0!y79amt}szd6ghXcX0RT^T^|c3@jbax^At1^-Akg-JC-o)wVARvUr5f&V3LN zC+dXw2_@xXOS!O+5o++nMhG_w6|ekGT5WOkYYpZ7Lk1O2<4T=-_$VpAe)U|Ni;23d ze#*zIn;|2B&_jl*hygE%-+dueTExDbyx~5%T(?kKX=cp<-~Q>)9_-XEO@JZSQjhv zyQ?c}!X3y^98}<}c`d!Z>+fOrUS7YA_~oQ+<4T)^XppM9rnh7uM!tvzbyola$@Qpr z^`-bH+Thd5`aEaV?~l7!nDYB1vcj%DfkBUF?nhx|z+BDj3bBp1C1&O2nfi;ULG~e^ z;Kd>T47~pLovO>!bT^+vFZxFl{gOL%9ne z?#x9PRlxXoAt$*A9{;61sRs6RnC3OhaD%HT>N6d(K6NDQ+sZ+JB_Y+Uun%zDRmC?syfct7VaOg*~-60j3m3BwWIUzSgf=}8~+&ZUvLdm*7G@7vZFZ3&4Ht8F2}$V(#XJCV#jSN1TIpB~}(~UHfxG(mA-akdzgn!Cmf}9B^(8$ZmX{ zq&R$-O=A}-$p;a{^WfeXz6M$mYAiU#0i|82N#uMBrsBwMWr~MCIop-b3ZKDiK06#Z z)mTjwMJ@0nK6J?cq+~Gn8D6Vm0S(DyE~kwhT@6O>IXteM{xF!E40Rk?P>_S{YZt{) z8srN_BwAa@d%lbx`t)^+0pHZNx0&1PFsKM27g5B1)$O><@C z4eVLn*VhlB`q^O>gBP_2Vwh8HPZ*3Y7r){pbGO-1^yik~)O>%%FQK|Mghn$7i4&52 zT?bGsJ9@OtORZ~^zxJ`+=u5lq*NtjDiJyV{Pyt91R(+zkadX?ck8i+i*7iEpBEgs( zRW6Ho(k=*NCnSvvhc#)i&KdrlS@$8?RV9qdqmCh)HHb*B_4e{QXKKo!F12A=;p(4a zQgj$)yW_`57NDeTH^QSwMaoR|Q|WhNkmX`RvOLOS=d7?)4I6Af$Ev9bgClEBN12Ct zbb(Ae`6{pZ{={EpC3D;g6c01nA8Tp^dU^ToqD9&J?ikvvuZWQ{|B|+)!_^@ptiGi~ zens0PWPw*1pbJXG>F#~F^l%_b&-H0Ko8t^?lrWZekB=en;Dh zn@j_~&SEs}Mps=4Gq+uEy_ySiOg{2$1a4g&f;|{vkbs z_hwwL420?LM-Uyn0UfNZY{p55#rb-9!NR3RldW4{6t~A~$E_C^%Z2$5?sI=IAuJ*y zB3rT_<*up7Jzwq7%c@bFQZ;p%qI%OdJ`i*j1wrp1X`^=)qX5GtG{Da1+005y0R z*}?(Gm9^>jPWRPV-qnu|caSUVd=d<{c+}9m22_i5JN7SC(pSyNaIX|Eh zEwQbS#Bn5(hsw03YY-ozl|V~I7rC=Kc1H%z^;t%lLcBnBeh^8sT^>z|is_z=kaqy~ z3}dU(A9`M0J|tuA$-C~%U9q}or(SJLBxVW<9k(+d$jLi^>PF6Tkd<#e=itoGhQj{38)VUxI}B$gUqO zlGuf6-g~g-Wf+cbv%)d|gj&{W#A#YuKQ)0&4n-!^!t4&Zf@%Sb4`t4U|HI3i|Lf|q z|4o56jvPIDG=%%5iK>QLAP=MK$`>e}S-E9c{E z$ZXzd6_G5lXPa-2X>nKAEBjJXT8@9EJD_|^Kt)r=;4i%JScF}ji`w{HC7}hW3PL0*{X!|mBv~jc|I2l{J7a5SXSmG)oAWgt>;F6(%{pMA5bhV;qd$e1ex05F8AY&i zs!%FLgI;YMNx!Ix%X6P^l?oO#NR(Y=S1fuUVA#)R3NM5QAjmwD55S%B3QClH*OfIo6HBr|@S*oOh|Sk{qj zDS|?Yj@r2iOdb@>h$D#%A0=Iyyns?fs@UrV42*ZmoFi;bMoZ>L+U!N8KPjd{p}L-9 zgX4)ruZoI7@qU+?Jh!=()gsG8Lhj)sceX+?!LxCrYc)q*7YL*b>E!a7{nzgxru<%T z=S+hZv7XXxfjlNFxg82F5f{eGq6~|hm^hoO=*4Lav7yEWi;{Q+5E7-^S6L~*G(A1- z%)rMHzB#5NWQbz9i3rtkbt;Jxo1~l=bcciDy2Q@2h0e$oOeB#BB9w%?2^Ch?SDx0L z3Q5SwvT~T#-Zre%ZVQ%qO?YyYzenzE4vNp-NNUrK8kO%#2-!C%0r>)zBX>81>n7Yf z8p7?~0d)&x4O&P8ketw2h1=KcG{u!Az6CNm_Y=lB4^!O!a1U6Wh{^PG)RH~K=spu$ zQ`T6_`#|ZdE*!UG#0nY{1chiJ70QA>}Bs%jb>Dx%PT57&ZEVhV$=JPK`j{@W|`p|T3-su?Ctk;rPKiQ z@Pq-tCPIK316xvU5zt2VL~Svos6{8XHjzdq)#2Co5wSjydL%zwYHJyv-?x*&9dl~r zc!R|X6mlS^)1>lzYa6l0ChImr9rmOAeW0RY2Cm7)nKArZ(IY^(AWm<&3-6MHd1Y#- zr{(nE;H7q42e2eR;>!5j#z1`f!T@?kMm8%Gz+6Zs1z^}wVwrK@y(2k>&$?;+fw>3o&7hI8K5t8oN=D6Z7qK9 zbUChenTxmkFjAZY{SZ($&3VMbWA|lV(UUtE=_j2Bsk>Uw#B}ysMslcEnudl35ETmZ zZWq(8B@f*^Gv~T?1N%(3F3@!=BuVCb^txx&+R~6RUVL?`C_nqY&uBXyJh2*W{XRC* zi!+>?UE_`(#A;jyM^i9&VtMy5EMhoVIg{k@3B#ke&+2mj0L`uKKDuW5LPb%p{JI+# zm5xl|I=VL;o#$~*<8+0N77Db#cLZJ`7RaG6KIO(85b(`2-TM?B!=(2z+9;nk3K&1Z zfeWDp!--CXyw(+={uk1PUB3j?<7*Ha3^1p~qZol}V${esVYf}=T2)XyBAB+uZ7FeW zg;=A?a^`l0^V%b@;hk5O#(?{P09orO!gIuldVvbGyV#>8eQ!k;R--Gt=secn*RJ+& z^K*}E;BID$XC9%i#`Q^?RA@G?0*TZ}b7}Pn2Sr}bqNOnyG_z6Dr*<*>RmRjOb?#Mv z2W>-RVJOtJ$1BbBxc%_`rE;$sY?~zz4rLffFGs5c>)ye^>(2H#BlVc_6O{R9?;WkM ze>{I>SFmj!+vNm)GndsyE?md7r7^*3*2k#0%((aEve8g=rKF4#3(K92&0(6?d1_Gw~twj@|)kP92e$1sW#H)592QKB?E>{fy5;6zz2T!V0k zLmwksd*%gCD+2KoesD*wH%#9aELJnmi=dHm0jNCI1pcA*nMxX{9zSK8I_vaKc`Ehj zX%yRJin9-_xkM?1Y3In2rO9 zRChhIUxg3zmwkb7;Y8AlFmP6Lbl%!Rb^9w$=~Qso4q>A)MGO5AbJnC6W8&^V^`$ry2UxI z0g?4-eSN_J2&IgZu+e1EeW4Bj+9+U%DY zvs^!_(Z?4Qlo*sJD1==$PKdJ1%Tg*X%ZS?7n6URuL(8?VkKMHV#`o|Ij7?2%8$xi9 zcYCA?{`RfcH=OeR^bZEk03g3gMy-lBL@XiF1GDvtQz#;ceH1wG4YH8}mxj|&i!Mx; zCSBx?n7N(`@fx4eLbIn{?wTv-(pemJv}U5Hp?tQbMS9*Nrozm_CCB^xBAngmpQ*^C zQiR)LKNoXxXSJ0YKpA*ElbtRg?7gJKM;@O8hKgpk+YO=K+4m<6a;r55UOJhI_DxfK zpkr`rjiTpae=Jo%Z%dD+EgW2F?2@u}?kx$jcI^F>*g4e}5_5+vQl3^YG{2LAqH^3aQbZrQ~s&B)P>0a;b7^DrvirZH&Dp!RydUS;YPt52*cR*~|joUqsH z{U#Ewg}ylaBg2#LJjWBksKi$%?luO^VR(D8iV_Z|ViVLX0=qA*{Rk*|8_AON!kW{x z-F4h5%pkUIaqLarbDm9cY2}=6yMpz-*_$tq_X@s0lG}TrarR-KG2gq6*ZX&JznZU* z;l!~RJ`~4s1WB3Rj5KLMfB}LjyV#RzssHtpKLLRm;r{>XI#QW? z#|5xtr&?t0dByi1aKyiUqBW&AQHWgm2LzG-AFumwB6S!j(4LWVoctTx7`I93;0FKs zsQ)g0*?~U%P5^`eO#R-gkYUb0U=M(x^07`xS94v33ChOa9zWMBN59i{3iM9zSUn}c?s zS*uit=sdS%h7#QyQ32QB0NeQ!bRPBVYY zpKEFHj~gd{ulGyabmaOwDk*a!1J43w{%gI4*ng@d1=a(-QT%PBKrqsO61o}pk(~U# z?dV5CAP19V++%@XH18GiEZ^Qvk}(iJBS+wyvY8qlF#eUu_i}va&_D%!f`*;Q+VFiO zWVek-_#x!=d0U`&E@oxcz@eX+y=Lg1#04L^@0@3^@kR1B^mUSCq+>#=(J!1m<~+@w zfQl!xbhmY7AxOl z`JgKZASO=2(730__9X~&^XWNYv25mx_HrD~ z2a3{%W;x3-r*zCkgBKXQ%TA+;KQv>d(u5{VIsDaxg-7iP&SIJ)R-z7}-9;z)k2EGQ zP9{Y}(g$j7`3?Z&q z0O50{U7U*|SjRJ}cSEdns#-Q)MU2fJG;B;Fj0v{#_Fc^sE0h$|ax%`oG2BHVe8aR` z3fBR2wC2n^f3@81;7Yn)wLEm_LIkEGSDOTmA?Ruy3+2?hccjNBh2OCD3E6${L&h9MV z`Ph;zEKIpKV(47}Ml%4@%4>!Pi#N(g4t=@z2`%Xrrl6mDU$Em7<8+c?QctOIhXB9M zGh0@qAMB{!DQk=@MV?iIcJ>iSqQ)lmwmkAS$`8x5yA|1gfQ~NN>r#uMr`cqtCBE1? zpc-9XzKwKjp-=XTv+gy%6Q^X!SB{ZXdD=U5W4t#+&&rAgz=Y1`;HBqBkLaWgq!{=}Fz|KE2qA{Y_{tS>AQH_MVfm5U8TM!&yX6dZ!2TQL5} za&g+jkJ&i|l*$i8&ZXA2@Wctc=%}4~WTYG?Y(Dj~cB8J_QN7saa-~tu2UT?S#x$ot zFO9z%TDy1Z%=k>wH8C?&dirX=pfR;aM~~_(#}i883?X5*-cu?ywrQ~G|FPm}5A3p6 zT7yeUDwa+tsj9!|pC8MObkU7ub?NcOtq$+pAQhWMp+MHe1F-jFsup^(3Py(6) zHdzOxflyNR-t8oSNx71wwvWJ^8>RDiMMR$}WRinGbs;~aX%Z>DE^=i_NCc$*O-@F2ECUl1Ie?`#ioPf@SRfG8$sSL|)5mgr%TledU#VyX}ef9x%De(s1=ccXkUQPXL{oO%`)@Bp(4xvrDP*$^?QS2piHxMr3RDAT{qm(tPkN2w!Dvmf4oP_?Pg~g7t&wQVt_BU#wf>p`e5VXi?Q*WxZMUyc1Xo_{d zR2pr}-s7icPCpxkpDV&cBtD>Ay_sj#YlSbCBjPauYff(Svo*x8ZeDA2ZaCQiGl2W9 zTUChY_L)3l^gmWd(8U>V-dx7G9pm97zc1=c)!ZvOpR?pfY+`lAD`VDwif^ZmZ&pZr ztHpI}mcV(}?HOK@c3xk|*eJRRB{S4ORggcwgxdDEtamK>zclYF3X6C)3VsYdi^Z-u z7e?rAZRj8=^Au+XhiFyHT-4Ug>}*XxkvOf^9;R)Y_%LonqAmD717FwTTzf1oC6!%U zyX;h{(`JcYi3E)F92Fx;H3g`_7uosw+y0i*(CeeGIc*hdS>Ha+zo|^RFg#D%E@ava zBwZfPBH?-6>XES(`?Yp8T*qBEtsf(sI(*90pH5GIY1UgEyhLD0aJD#Q9P05{T=3Hy zUP>LeV{-w9qKp-oKjl}5h$wu5}kDcSJq*s?L^pr4M}hqpg@PSTSaA2I~{mN zo^$WiG2@$8OrV#GU>xwIi`?jM*d=M~SzBbP3u5!|@XFr8uDM+t=;VOrtF5N+SVlks zgq5OE{+_-e7hG6I(*c8)SibqX@nyWFF$7O2Djkh_F96X3*vzS#%L!`}ZJ8lZCTzyV zjV=>-Kg}SiP>G^^m0p?Na6 zLkqMqx0KV|HrkuCjh#3)Ha3QNoU?qZx$Nwu{XiVHYi4JY)fx+5@oA(qSKc6u^UzGi z+12yxnH+jQ#SlYZxU}26pFppZ7pZgM208hF^C!UHPnJx*A1=39 z)O)jeU>9Ive<8z-n6VV7L)~BhLdT|>a}sp_sx6=d_!n44e|5q(@A?<2|92!^UHPlH z_(kQ)N6r8x{<|iD3QVV%>iVr6h~EGgwi;Z6HFuhAr}cl=A@{_PF~shX5DS2P2cNsot$bzb5OZDdNa8$m;AW&vt5mB)`_#I4 zz)Ngm;N~hTv}cI5@flF4z@_sfmv8e#Gwh?F6KCW~V%nsv?SDpI`tXQp{`GM~$AH$Y z(mQ>6=u7<6i=3F_i)xtm*!iQPNOfhyK_)l9(P+owIvF0E`Eu3p1a_;E>8H#&V}2m+ zHw#&btE9ep`aEQ$#=T>D-hAtAeax1f{&qJ`YgL(K^i6#0jTC9cIqJ?CiZ1+Drh|s@ zx1qwgZoZbRWhBSk>>YomVIa3FE5Fxkl@eQUKd===IcoI{f?et%M%9yw zx}%%Prg5=aB)qv8J1{^y{hyfCLZZ^Fhlyd{Ll{ot3dwm)4Kti=Mr~cJe=cdJ2ck*| zF&8QlD#mVUsVMm7>({SqEgnrF+#b9eFwuuh&)J+te}Gl$tT<5a z+y%K0sGF~UDJU!q;r?JtYm0rD5tFsM#Fq0XaRG6;LXoImU29N z^k_|}3{434?*5zy{r<|nGSz6WbK&(_;=+7p<@`$Y{=Tg+|Ko9_*lt|O&(DudV_$0C z1MZsU=6frv__G=ZQ&*}K7^~*`H6?6XNWwc?k9^=r*cwN;=E*CH~{Q~@0Q-rT35imra%Yv7JY);#Jer(BCT~h<=J&5#KOp+P&+nCni z&}eJWQ=m2AtdEg5baNA*Jbt&hyu2K$$HG&EYy6u|0hFl7Bt+934d)r24MMTBIJqh= zl}C-$F?JFrZjx5@rNYVl%NdKfHQ)O{E3mY-gVjwBP28JnsFfibRDjVij z;mqcRf=i;@$Mze>qY*9u#aCm+VZgc5qicYQ+Vm`V87lso(BHrRL;X)20V$GE z1_8*T|BHt{+BwDKaMyHxrfVXfT=+K1`rOWZY> zJDa@?oTj(8YU4V@NI3C;Q*@hzpz({9_cDlbi8ZdXoklWE?kAdmD|~`d(oWjP!1*%$ zpxU1evBg6cmL})#={-MRfsZDC^TyPz(V3eD2CaJH$Qu)226<_2uF&o2A^Nf-JkRY` zwM|_TS(~n1yWHQ86yR!zC4`;;5`t0CEdO{wZ)6n(=bberQ*t`c^?Co}!QNdeO2tF+Dve_w>lojBHFk<>n|_Fm2{` zdfrKK#{~yq4ZLAMN2)}mq!(IFB92?O4=;pn=pZNZvZi0X5ziD^!{_hzvg+ln)?^Kqx2kI2m$~x1_IW^g zzvuH>F1c1&`zQ*%06jv{_|x2ntL)}`{eX-oJiKn*3+N2#M>)fwEQ=A@lK#;O!GV4e z#J?3C)`xfcU5i)R&3lcf=@!XJqZkOnxB9y8s3Q|uJdErGLu$Fi6}PP>ps3QI#TQs% zU89LIU78Di&8MoWa@EZC=T!F{=k?aC<2`HzDiE(p7blO?)=d@RN55uG%|E(Tm3}Iw zILzYqZS8gERy|NvsLf&Ar@RCoj*<<4-aCoJRu!P15Vw%nZy2AAOIvc9m`RRQhi?hBzYe3CF@jK{xoE(NmUv1UU@?Tku;X2pA(0}(mEg28+i zEJ=mHb&J0?1=vhNHBT2JQ&BL*J$dEI*$iAq%N1X689&)3c9^#HZ&dd+dd72M6phdY z9Rn=yW0%2p+br|`#<{y9sSv7OJ6)&jvL;~|DF;H2bm-Pw>uJ=3rP zC+!?6$^hR7-4{*ut#U$c!&JzZ(FYTu&`o>txyV-f_nu=OB+_Kv*1|{)Zyr|A3}^@! z_l4=F5*|*7O=0y7}c~a?g*Ihh?g&-bQ%l3Rixb2O)9MbAaq# z6iAOEdG?cvx+;u;_C7hY?{$;q_&dhSmqcjSZ_e80jCe0UR@Y2-a~;LnybRU7#T}lZ zB?tr7g4_I5f3n5+!}N#RlXwYB1Ct9wMmahLGdCsL<_NkZ_Qt7m%v=EhkH^}Qpb5A! z4wOggX3gLC<*9UMx`|y%@=a0M?J1De9uV}JS5+;mZ?4}8HwNR&2xw**UJ)4k6WAKl z6z*fBS9d&3W=VC8XX(+10VXZ-&}DQh8osKM06vIk7nyhdl5 zvT5AZqlm$qz5BC1+nBuoMe{WeeG0>kO-z^E19U8gnR#c}78PAxdIrg%r+=k0C%f8^ zKgjC(GC-dmg!D!k#KI+Xzbdg;95LF)aLCV^5#iZ!cJNT5Z(k-?r-T~KyOa-On zQo>c$+6%vdz)LfSGjW9kWs9lx>w)hC&J(W@>;UBd)<{N{ z4!qI9+xp=0HBCSDM6wTWkWRx?_r`kgf}f!45&>8*ggu1!6OhIOOi3J23KN5Xh|Jm4 zXH4HmM_#nK31h(0$;iiSh?bD}-bY3w?|LN3$i&>iDTxil=xnxl&-eF(m}=2{W-*3K zLGC!^<<$ZKoixFSCs(_&3(N15oZ%g&O6l)8iN#+#Ito_%9-K36?|)O{qrcV%eZg+_ zP<^quei9pux%tBM3_|thKFEye9kW7YhqQZ&ajq|eYd*@o2|xi%a8rH!R1OTLgv7Vr zKFE_OkXVs)CF%T$*D#-9H*A)C9D<;^TwP$M;;Ob?L!sPfeV!)W zF&`WxpIC3?rLPO7Rj|5aP-i{WuzA#$X0h}66!sdbkC}GRkjS^3hL?=A9`l`dYbZL2 zceeqgr^#J*e54BaBr9aJ2k3)_kyqOW2}OnFlwU))Cz@8PNlm#zB5BoThF4h{1*Kkf zvUAiJW1O7wlatF=#GI65>hlN60T!9y&=Aj}9>$#>UtXu;7%j(Y=~gc0`TKO} zf$Umyq0!~_*tmCeg|t!7n1R>KVvj8KcV4yL?vC>XYsdzw0k={O;tRrZjZ-)M8r5Y= zk{)uJUu<8Uua&MjG0Dg4@}a+$02Fy=zz4LIjo9(Uln2{wh?;cAWG7rgV4OGeGMRbf z=F2j(=wne^dNNsZb^dA%lljb9YNdL#*MvI8LDR68eRpbfEWW+(y=!a7@mvXcr)E1t zEr2X|xy5b{8UI3Z_@XxP5e$;NvynI{lKD!7dmPH!U`e285sw#hT5x!Hq5Y@*I-5_p*|*OYP#iUUoSXYrUC)7kCJ&d4 zpw-TOb#Qi`f9_1sAB9&qYI4VyVb&doy}ikByG_V|Q8dCA8jTWdyGE^N_a;LKPvLBg z&;5xQ^|Z?2C0SReTZvK*-QiQG3x!yHZ)j#cZ-`EMOz#f@SA`$N_)^= zZ#FA<4w6{NyO?(6?A(HTq-Dv~IU5%PNqumMB2um04p9wQjb}`&s-r_5QM%7+OVTG( zlWN~}W&Dvy@a^&WCy?=$8;i*-_%-aR-w~vW57=M9GLA%S5ehbvM3U*A!ob~I0>XaJ z3&86P)psPP7_JH%AG+WF*k>pmdIEgiyfX1eE^I>dU>wCK3#PTPd;K23FVAYB2F-5` zr>^31#WzKO&d)B$K5a}F1yfLz@WLbv`UTr~Gk}hCOcPi9SLeaJ`Fh?T5U0-+&5&fIxvfjt4@SgA+yf8jSvmWf1YVZJQTiq_W3=3;A&Rt@lr^o$R6oQeyc>T9`iFWsuO zBNRM%R;aGkVWSJLS5#PU>M4-h=OHE3PVW+CV~{ul6E}^EEG*SNeGF|6Ka%60$w@*% zhOpk#W_IR42t>sS`brxqn(c};liKURj^SfX@A-i--zDZW8f?k=RLZtL>`KCa3sDB-MC7yD>$B+Y0X_BCSQhWkwO}^$mkwYqg>OJaK?#dMltN2(64AG zNFj%9r7hw)2m2`dU{{+rvw}g5icLak^&XKZp%$R^Q)~l>ZjYc~(-`nlG@QoITW)$p z*14XQ?#U;;fmdZu(HfENltZHv-fp>n?gTvt+ef5~nCtTnWYLC*1n;i(@c|9Q;6TjB zcQ21vub+D!@1YN`_97^>Qb_$LGWVa+qn zO7wnEA4bpj^x3nk#2V6;l!SyjhM%sJ*lSV~ONSjfchTRg$TpF;Y5fhU$<6=kSFe_q zx5!&6He*u4l>o1!V#PPbn!v7Qr7;}esk&R_$0s4Y%$va<%5Jh|J_Hjr76ceRR}{Of zvmakW)rdCXYuvr}!MZ6`fP-tPGG2pEih;o=*DD7$>tg-ZJagA|bO zSIFi~Icz)k&!Pmco&mI$yTZ;8_wK8uB-LlqAByofUk$*v967GjT3% zVAU>ly|BTxsnwMMHQSOFGWJG%#+f1JUmRzs&Dj+%5~jS=*B|#|*@={USL&TadQ|Ya zDAxMErJotuE*(kqx`ZFYgMUSg=)Nd(*A2SOEw!t7zZGS{F5MT_Rb#g9f1TFl*-81r|}uVZWrCSw<0%QbcLGWL8;c*ezb|Ag;}vlXRHw=?S68>VO!XIwv)Vwl|j zstW&@8s}J266>~2QlLb@ngeMTBSjiydT+KD#XPZf#W1oxS6}hUd5R;_+3zPi;`in? zmmWf)-r=)|p_9N^(%!hpZd>^H8ooDrk~;qOG2|JF+#D|W&`G7Xj?MWfGcpukpg75T zQeT|6&u8OxrZLAvyZ|gO!4lofJZ_u_KxPh>$e9^Kg*W%VT3rPjM z3~RoBW&u`?l0Hl-p|~37tM`t|r%@s6$J?+|9J?11lA?A`*lk4;t3^ykF7cD4JtZM{ z3|`w$BHK0MMl!F>kuBV1@YI?83-c8yRNOHac<#>kg%GzfS(a+eSYnH~Q6o0R>Y|L# z9XYu0zH0>EJK+!V`;k`yNTshI-DQg`o!L{lU`NVa35mOl*+x%>p$rbZ6Ru=N#2EaU z-nYl#-cc79Y?B-Jc9quMK&DP}OrzT8O+wmA!S~T6%!4g0%z$-4c>SX0u3es19H%6Z za%2tnvaNHiXp%ZFWzK9;izw5IPV<-eVsi*}BEI*)z8%}waN8N3H7`5kAAs08423fC zLAHn79zC)5{30Xj`Yj!~-`}hme>kM*(Tb~}-l81LpQljo#XBZbKb(*v`q|NMeR1%-6%J1uxraQBd>1?o!)!`K$t{x)Rgj2Iv-1M_VfT1@`iu#yn#Kkt z*5^j$FL}v=D-33Na^KZ@Am(Q|9aCt#EtwT%b|heMG33!UejF3hcJM>mSi1{8hq$n< zmJ;=X_2+u@osU!v>swXLBX>u z^EPMk7@Ee6^73__uTw4IOS78C?ecOq9x5os88u{XS5?HBGZOtP9M%MEd$c5LRBW}1KV>eQ)hM?rHrqt+1-Iu^ zh&6hKPcJAm7B601p41{0WNumqsI7+9Q;04O3=DMEO+}hGX>ev3n++z#&fqDwjfQed zX0yu6HTslGP8@mD=8~N8!6T)y*m}LihhHF$&d?)&<$8zWHeozkxHL;MQ*Q7z9NxAf zeO468!s@Ud*2H|?z;g0M7!$hfncIyV*fjakaQxs<4(xNPT-;t#V@Rm)yx18yk)O;N zPaXOck*8;F(9qu;uBA5RP_8DxDnQHqIJam8%_ySIfBgMrXn|ekta4o0~bPrMQBo<8Yc|3)%O}sKhCY8zsoquBIntj|uZccUCnnmsk*; z8&=J+9fShVqOQ3}Yh~qXcYlA&R%%@OX5F)LyXl%2m|P47lf{E6ILPj0YMOeY!sb2$ zgT8HM;@2nBM)T{wG>+N%iFkkg;1T(?fl_>cqvkw+Z%G~KWxBNqbMCvNP^YC5jWADI zBJNXTw3z8%w)y#`h2s^gK*Z-^Fr+I#7aucKxMiHF>jM2u2xvJ!S}1Q$_7!rQGpUae zUarJ8wJkdq_Lt%tYiHQv&we(bs3fkCqfq##@x6wYlWJVtq822VWX_~%(ats{+|}DJ zhqd)xS@23v%EM%8i zyxX`cb5*@Hw}ED3Zjs7V%672?DGe&pu1^C8{1GESG znrkR*>j;Zubf|H-Ct_bTv(I4X+aB}D9}#0*G8sz9-W+o!VHv5qz~uqwMSO`ntH z_(kLdv;edE4oBLVAE)PfKBxYszkEF`$+`OHIL4{Un|Q#AIxP&8q0}gHvC2taRmEk) zXk5N7=E<=gGtFjxux{xErBbq=KC4mc*wyg;vHC{GPgZm}Xd8~UUGS3ed%+iMCyb1Z zolZOd(9&~TcdEDWT`CLIcg1x-+QoM^TDbH6F*b_2kdRO2u7Y+In*_DjYE#T~vQ!_=4RRnxvi{;kj4i-Ruh8v5ZsI^ zFn8?6ei$lD5hI5>(w=Y~?x;+g@i$iPbKRJ0(MdE_$rb1|`81bNw)%0gdRqDHs7HN@ zOh@!0PLv})&w5;_H$N;O=D=-8b=r`t+)I+4XQ9XE_tq6R7gO|v#3-woo-qBRpv~a2LYAlWTF$HMgqnFzV zgvQ0X+vn|G?&Ldn3{?fQ?cAL=??;qpwZ|Ro5)87k=#1Y8d>`DycrOAzUZKNv)w}T4 zibpmcqG8)^c4N2SokYVO&D#-h2Pdk4E<3Eqc;Js$r>$$;n%{%}VJQ79U|3z!bvAZ; zd$*3MMMuIl>oGo+dy>u1r`}X_^V)9}*e!)P*R=(VZ53~o@X3=R1{nnOuh}V~(dY&W z3nt^mCzvLQx|n+$evqE;EnhV(qw**4bBhOj9qDuCxGv#7Yry?{AM5nF{R-s9y6l_g zq`^0+-M(n12F|@jwMj2-sGkKO?sm0=`B_4Xz3e`(NY7fUt2Lg>mHTNwIHu5WneD3) za?7^3w4-7awbY(}poyua{bqpdcm3JcZ`|_t)rzHLD2pJe-8+B{k|*S+W3VIwC)~xk z?~1X9 z4qmMEKlC^LKY?oYdHlx|e@lcMv0pZ{!0q}iQ4#ar-x49({0ztrMeiEtjT9<7QXmoh z$d%v92JlATTjc%=KO%b(^x-wsbn<+HJS{oU-Pgn0w}<||lT+y=J71X&{>+WvB~IJbkg=)`dAWj~8Cm&` zk}u(Yv!!om=ZMEnObR-6Jds%k|q7e9XF;4(L4 za|hpN-tSVhzU2vze;3pDWY29Op7%_`{uk*F-U$jut+tq$nlh0BkyDE}(e(z0-0QKJ zCka|;XaERY(!LT>t@bgOF6i?u)}P(Pr@;nSzK_w1JL$AOrlWn^s>MFw)NgsKSQURT z=V!nJqr2=Bhqd#Nu=!3H@T$sjs;a~V!lTuCBJYPy>F`Ywp_*4`)_^ansgP}ySLNls z%!2%Uxmwv!KvKQ3esdXP{Z24fWX9emG~W2pq3J>YvU2Dt-qgL$}w;0n}zYq`L3};0`F~c4NRxa z8jikr-Pb3I&O|G(nL4iuidVxwFor<7mI51_Xj>1Hz_?Fuko$QD@vUmeTyzQM+Iia#4RG)j%~E2cyX|SbrXo zlX|>D-Fa}|jed&HY%N23x{e$%C>t0HM5<^FqLnTEpBbYOz%3RRyLfq*wQM>-4{^hB&2@pZz2VW83j9jz5 zU}t6XT(en{$u(X15(j z@%_5)ck1+Yo!d5Fh>?bkMO9obIN>5@{Pjz)C$XcML3E!D_eM2oO{D8@GoQ!YTi7gg zrB6G2E^Cf69N!u!)|MrsvR-`VbxEHflSqc+uRXl`QEpveOm&~7>=MCGe5 z#Ksdpa*v`=X<0(#hDpxxW2sCoX;6eA6(2eI`hgm|z_^JY5v@W_V|m^3`Fr)`yh8q4 zb>HFDRQCOgjTumMMg*0vAd#k00tf;kC>@brL=++P-bILIgaIi^uY%IM6loEnGJy0B zLLdkzNR8A`lecgDmfw4Cy+7cwT+2X`bI(0x@6Z11&pvl$k30^?5n`LGs&Bu}m~#B8 zo}xYr(Icl#*PIl7M;*bAeYy27Jxj$EdK5giE40&`i`Aw!HX@GJD|Aynl7j~iF$74!nx&PQv#zSG(69~WLa}8o_l_@=IFbsRQpzs86xqOH+kiw4&GzN9}7hd zpHOSnp5Q-q-YO`9{nP^vY6zyLhrPt2?`d$`gR&%xW0pQXq~POWtmm`bSiP7m{q!!I zPiSmUr3Gd}6aq7s4iT4HFiNN60!M8o*aMW5VqrA=U+Y$<0^zP5#QM;APgmEkTzEt2 zh{Z*xxq5dK5=EMA^J|h3Py_)mor4QZ5J$eKt3RXm>~RC@WglYLsPq=RD!d&KYNwfa@}2rTuP5fMkdEN1OTt_lRP?8$BZrvT`>C_;FZo);2Xd|(f6fQr);4pYD@NwLBd`dHt+U@ z9I$9>(~-p79ZK@E5EX3&QBI)=~F@fAP6Sh9V;#w4gA4I` zSGc6c_~>kph}>CP8*wNR7@7>|&V661JQ*4u_H@WLY2$AAJs|f>-Nr!)4-dxFEA98TPBA6U2#4bV{Wp z(La2yKTpp&9pmHEWoxE=0_Lr7S zyWQf_((KptkchPLnKJ+~Gn&|(;$pj0?K0`KN*yJsY>os^jiji>*&$8bzp=~IKMj!Gx3n>U40wYlw|mlwHPYYaOY`YeqK z*d|YPrpnf1Wzp!YdYQQ7t@U_bws1lCrj^w<)kK9j>siX)4wSzKZ#&uAszO_#hRgN2 z{Zw9FUTdm6SER$edy2YSrDzF>%$F};hTd-zif;L_G=@{p=?fOij*g6&USNG*GrqdR z#>pvCQ)DG(vf*;>JCtO&J2*J75YXb{(T9T@tgBd#279couAZwBm6bIsm20#p_nITy zTVzy?7tB^EpS@*(hY<>OD(KI)l`yTO?(FQ0jd#FN1wB1IWrXLAmMj)iaIMBgeYPCB zh>0%h?$4Qi?DkC-YT@QhwjZr;K3Gr;B+1L8sG5uQyLGeKU;ln3M`#bYLRMRvs=4Vt z&)%geHDk=rA7;+aM#hz}dT@6IMC~+Q)Ua^jsSc#HT%`TM49Fc%J?+kmfo_{jTQ{kB zvxcvqNoezjTd;Qgxzx%&G=v(%TWr=j8s^`ZyeiFMc=n|Ze>CKIdw=}MO&)&Hfr-`R ze+NsTOw!lUQ8Q1!b&ve{^WRX@cWf^}GI^jQSqiP7U?G;w8>?>DuD9~t%H*$xdnR#N z`hrfUYViD2Q>pqeZ^d$!ytoRIalze-YbO z*1CaRmekAB|M;cM$t+RxMxMSp98%P3CC0|Z;Tima64l&hVSO1t!j5uyk)Z-$$`3`IO<^nDbC3NjoSP~qR>~Ie2O(M$q z$iCLeweoKK{8?3T@UOJ3t*s}TFA@_^kB_fn9w=w@>72 zcDuoUSN&aquUU4ac((+l0V}I$8YgRFPj&f<2V|>n#Yd_StFG&bLXPfI@;S~Zre1P4Bd$GKceAOUQ%+i#Yz{77V1qd6VVT| zv$NGVu!6C=SS)_WRB#<1{dj7q*j2i)!iF;&=R@M!=CldmP@5x>E)Ew@I&FoE(lvib zg?JvkP1B#5cA_E5Zx*h-4^&vaFgh|mZsGRPkYpuqUU9Xo{A*FnF-gATA^BC`rAjYO zo8KMue9?qcU0Pf7R3wC6&*+I#WKcV_r$2j2eXG%HZ@;s)T1*Uy>U-smw=9uPHH*T{ z!69sRoF)9m;U~_G`!e+fzi-^ia{Dv<#sGmJq_*U{|7U2@+*Teqc4jG&yQdVBSUqBL z=B3|1j&px}kR%UlEVF^xmQ+fI*0d=iG_uQB7ZBg(l1{&ku=r|?TgGB zOjy3~%WYcs{`5rk z|Cv$r?5}IDLr{F)SW>nz`PM}0my-(WHEhJYPpU-1d0^=e^Hw?uy;RLDDA0hS znWg>sSgUdqGWpJW&jfMLfc@<%SC0E)QMMCtT!0w2dY&|N$6xEffdee~l=1j}osY6S zZ>v3ir=v>?zEte?YUwYdjeO?a495ocBP-fXk<34rTQTL$t@L7zCJK!^`e)Ua$e9nl zdp9?ITk7Ss?-s48Fq`5Q%6J1#Cwl~318ExuuGTj^@wBmkE5 z0pRf(e((LN+G?%q>-Us_*y3+tk%p^pbyJVmKFj`@Z>B|~#i}=8JGPw1MJ!SOLcGw< zi7X?L`PupTzR}Uqmy{-d;kI?W6G4Ts|AdRs2}Zks0P`N(ipftO>Fgf?T%q%=We($Y zIn3f$ej=Z#(0AzYvUXEl^rpRr>HFu$r1YAUg<16kV|y1C3US>lYinz-Do0sn&^F*?vdwwnpz6#Z_qU>BZ*Z6wB2AATq)kC*Kz-CUF3o{YsWgl z5Ph1Hh?2Q}_GM68W*+|^EYBMmTm5(U-y=2#*F`qU3%XE^R)5T8tYOiL+%7M&rFq-7 zs+bDi>;$$VgC3!CU?`i;GZcsGL!Db4|dj%+gX(F!k`2vX6%em2LDF zGatrm^2K;KP$Ns%;J1%-`j3%3^{&D(0iSD}K-T1o(H#7rV|D&ROvnG3T7GgoR{89N z_9pDKmaD3;dN3HwD8t5=FA3Ar)1y3(oCr&wY9sUo4F=v+C@Cp{sxwaB>^0hLt>NOw3eXj)mkp9O~6kMCFySr|s)BMIF0f7#~)Xvhve(FnA zW$;LG)&Pe3%{{U=(Gb?hE%@8uH8eB;*}zrR`Hq9J^r`ZpQUYKfk-e^;{gaiA-u@YY z+((8={7jl{=+Kam+ZVWpb|)OX2R#e&XND*tA+-ypy4t!R52D6aq2=Y5oJ;zRTwGk7 z-P}qiD`7wcondQypPreC>~FHKl$Dnc9KDb`vb+26!toG&$F^G|72u%eNu&~ce*{le zc^%&)Ikh40pJ0p~yMx7Iot>PDCLN%}xo8gQw&5t(Bf&ia)_>Du4o?{h#TxFSm^`g- zeZ5E(Oy2QbiXJ^_K_PnTxhi+Pa+;QZwZlh)FOXx+PWBKE4@}$<3?vU&(%=hz^LtM* z`Hv$C)Nk=Mku&lwG)08~YKBqaHhC!a%>tF z8tciELdA{l#{tb*nE)!s92H4l`2E>6w$rD378V>0oEU=}U`CQsQ$=bz;1q8FE{4Lh z?%UXh4@ax0HW*AqKecmyot+(K6^tBwlttVf)=Js1<8UDO`u_Izu-@6Vms~qc=|={U zOcD?xL(<>11DQ(&xqss{_Ii-Argkx%J^aQ^N3D2UiG1a=Zj;{ywNLB(;j0lM+~(dE z7@Fqnd7{*~Fg>iT(&<^l1n+MZZ+dd0TE2xGDuPSNN~XB^qQe(CPlu>j#aLo_7zPFRo z$o{zj9@w#(XzX5I%UU9YmtiE0ypW zBxvUYCWsbg!|x#w`WCw(RfHc}@|Lg9B1!74f}8K^>O3q7h0k25)3O0OQsarvK_xy| zj)Cxbxf>gGzXT2-ju`(;DOfjJ#Y>JIv;ZPzePXAoA85tT6f7jVA(v}!H9i2kg)Hjw zszm}cfe>`T+@oS;q<^LF_hXlK>(;{qT>|dco~{S+0itUtrW^xO;~tXtc$D;;-7Ba1 z-*@?Ve;B?>NVd9&B}@O@m&8LL5=Ghg7#=+0|Dnk(+nHK|*}^re>>R*%DM3EOSY;wdc=)rUYFbV-?&NKe8QZ0IUs4uM_G}v zH)VFAch&==Ud+}*9Q2q{WV=Z8S6OvjZCrjLQ3wd)GXbi`X`I2bC+bwQEu{V^2=!vh z1W`Dxdhp>{6=FL(1tt+{!Py1T;!+2=_w+th8xG z=J#J5$ONngUL?gxYY0+(O`;<(Du@IBKd2t)-~Yy2AU#5Q0J$(&A>iL>EDW-c9s#Tf zqcytwD+mvi5;+fEkJd7hHdF<{(l4F}@risO>?$7fJU~|w;V4@E&?7BV6h6^W`tN8?? zTh1R3u>}|9TJRO%vzFAptD;Dj6*gPbOSPxj_X*w~@gi+v{m1ESU1?lQxCK89bZ$ma zXJ%q@beTz>tu~kR#M}NMROvyLfVK7Yys9dr!-o$;)ZuK{aSK?Lu^JDRN+$;};Sk_) z98Gg|*vG?z3?(pNr-oiACh4o!dZ0ExCR$Fsl@RA;VoG#Y^8iBxXtm5~C-#XTfa2&D z)z*vd8jt9&>P-QKMK=h%6H|N@S~8AmUqKfh%@b=nX7Ozl1m)remWA zhy#uklKT$1J#pGI<+f~NYZYr^ijZc#OJk_SH16G2ij@ryPjvp+a+)h{aL~SL55iY9 zB$|?7GXPElFC`|X`_q$?gutSLRyGJi;qII->%iMCE+ybm1Sv(@dUe zT}=hP10ie_qlFPf*n2zVF>JGCDKCC04CQ_bVwv%A?6;gt0K9g{B;0eU;D&opKgSl3 z6zws3?X}Lw&G%Spy}vLYoU!^ge_4o=t3a3H%atQsy!l0KD-dz`wQ_f~aVhQ!Ip(o3 z4=myck)G8Sbt$WVjg$O#sMRGyvd)q2M+FV9u-^G@+}Y8=B7BD;*j%=RY#5qDy95UR z&=aLI0xv&bY-P8Ki@oUf;Br-53|1uW+<^Vk+(A&c{E*d&C6Fq}KiFupiaStRSTbRKA-2cc0hX``9|p>o?l$pT)Aj*gDX93DoPID|QR zsmd{eIs393d~_p&cFhOfrQdymHJnhf>MU!oGj3a4hII@m`Ph2-ec*|hnVAi=CShI6 zUhe$v4$B9wER=9p2HYrs0V8h+Qi%p_RbkV)(nFV*_)N+26-r-JB zAjXY=1d&M{1Vl+P2C}BshjgcxS7Sf`268KMsoL|vVp*rjvxeV5 zqtON%d*2dLQlLtCW$X^%TYw-i8tRm~@tv}i$-wpK4h(EfouB?$+vCHS8cMui+EeHm zbQcZ}Xsfi)ZCQ{kN13*@d1v9$sFr zA(?#9Ly&T(i1`TYdTTSFSs;PzaGwAF- zJ8QAi0HjH(4&$(Df!y&SG&X-K6K=FDjk=k&vf&Rq2BIqowpeQuxHjN!BZ%Snqk8~Y zws=c7e`Yx0C(fUu8LF!o{y-zA4`FGj<`0F%62pMsdHD2bV@Ww!j#A_HMGw@|&kRXG z)8rF+2!wfhOGkns%t>!=lyF<`&!2CBgaS5h3MXQdM&wD4p7?InY#jHUehL_hFp(#z z>RA%gY9VX$zz;03FL(sXU%7JoQe4W=oMQuYcgiiRZ#HUQ+NAt*<+E&TY`W4M-pvqG zI;X|X`MV@@G&YMtWCi>p4Z7_ndX;<(@XRxoCeo=__Lg zyB&jR>Nx1O#3*}u-_+C;+&m!rL(#31YWYqCNH+qW1ehwA+#wj{`^yF!Oi* zya+3?JYdUI`Q8AT1gF!(0D~r{=J7e3GJS0bem7;*^KRxX`jXObYH4wfsKB2H7f^S< zYo15JE~~7}*5se6zd>65Nl%wQaS*#3@htk?}i&6daTt{Mp2BrCvUtLy29@(q2y}kNz zg7Vqdo-##1VS#N^Z~aY0!&+NyizhGUm~BT_mk`|3OVw(UQd^mXtZacQ-&r-;E=(rIql4594s8dd<$stiUOV7sDh!^GRD zmQ40Ffji|H#E9}&fBA@A0X`ZS+&?Zc1UDdaXnj>B@v4p3X^8PQxt@c~Kv=&=PpwUs z{&9D8?Ur&l1AI3OG4lXyjBR>`6t&8jw1sW0x4}U{qX7x+*(MM?g6&96U6*48>n1KP zUO$X*sR5~3t%C7YUVP~w)f2C3{3s=^v55x zeFxJjSiBXj%7+XS`TP5gNgl6KSLubNoc>LpO5X30)-^nt%$s?bfHU|1jWtwWWIr>& zH9hxn7n6}Ob(1?xQf^~diqW<6CrZ-a(~7qD+bJB)B$9jt4^?su2y4$bx2We&=V=fb zvMhdn9yWCQP|{ned09IRK#kODR?>Zwysk{{K})wmmi&sZh1_jrM~fo$^31g{f0NMz zdg#5GW_G?}3_DO%RyQI%H1kiF6Xiu%_%`U>1!INj1I4K)EZ9F4ssqcbrITs_ z%e*)tQS(W+{^nHK_a&?EhVB%6u^}!b=ss=)D+7TtS#(3nh1d^H39PMzw6w7S?mel^ z{$vUl8(Z&M8%w)*VteUuCPYl&&vM8Ip_qCdA_EQRWTI2rNyn zoed{gK;aP)_OJ)Oob5mb{hP1=l>C%8&->HFUY(<)e`XFV`hbl)7vr=1x$;R$ev=zN zmYA-X*Bm+&I?L1HhH4|q%R2L`me$wHOQ}v;@m_9j5LjhDeOh3rez|g9n$-5drZ+{I zik~~RQeOqD+2DDy(PWJ1x#J;E=n|OwFWdj@I8Pfv)dN$X37=l-%c?OxG{vTJSnCci z1PuN>eF!BC8#Gnh0lb;{55w(yB01o=#_9ub`RLGmrYDk?Z`b&S-sBs!WR_2Nfgu7< zJXR(q!2OUNMg@r9aN#cx`4B1LRY2?aD<)L)>vo(i>z?z-wDRW6kJ*ry##RFgee+7} zPf9prAY-61SFRoz5x?^Iy}sZ%LM0rZCP%*2uPc8Q4m#8koyr@zdV`m%!PNRLFbuCY zHzQ%u^Y+io%pg1t{rE5UR;rTD%}!$ng8(1{BdDV5#Pbmsr&Mn9cZ}X*sy~V3NBT6> zTYx#w@y;{*&jSY=?f3o?opJjP9az%q&D_lxZd=5Skh6SJ(33kW2F)*|qH_lodysLy zD=sQJ5r`*R2dU6%qaJd7f+!>RGw!MrdUl(b)2R$(X3CLzLDDw!z}m1I>tng2q)H2~ zPxBqHI`>W9=qQ$HkuQkTlNU#$zXz81a`=tM4_u&IEX4A1a@i24Oax9DbUjM0O7-yY zDA6zbCUI174M!U9}p05 zD6+q}p9-{|95L`_?Ft2_`y!jz+-tVqP>aDtpvSg zLX&@N5gsBpctv$8RTv(OVzj&G&a6XPuJf4DZfUWROcwb(X)Dn=UjYdhQS;)ilx3Zz zfNxct_-FA2i_}SS(YOPgf#dP?*7yCV;;d7R-{zYVKeR*IqtX4f`?>3#m5zjBL1V+b zc1L&jUbz_qT-(#fRoizRe1ckj~Ou_hOtTJ`S_qLGap+xYfNBxcsPPr>r1?# zIk*S7*lNe-X--Zgw#s~XL#O8cB{{jp2&{rO_vM>Yf-K!`M0TR&!Q=mCuSc|`=)w^< z43RA4k0BudgE8wl!esTO()^O*>C>n4%viM~ljS5{ugM$aNngDLNPU3$J}_#q+0k-x zA~4r5Hsqt#)+UFTWeJtXtYv`KMV4!HIkB{Z~2D@{{C%of)F`DzJ-j& zh@F;7ziSf5uDlj8{~)A!stkDjKuF)gKco?kquesTJp)%8ASofCe+dK2m9f*NacZ0- zCoe6X3y`d+qy(pEHY2^UzFtFeBZ(#?B-sA=il(E}iu@(t$R8XU$^n`I@N=BfiTHg0 z6u{=z5(PFKf{m8DBBBnx0KW@cE(=U9($LV!cZTtH*I9U7z04`_3JYL->bMopNd5QYKA5%4|dLCT^?msQ ziJ#w`M}g$vFb|6kOGX$iXx1>GQxYq;kW-{STH#Il5gJhuo3+IBj&SVO=rwrj^`Jj3%4K z0oBZf=)sV*85_T9-w^8h+1@tm-}%%kMFuABNuzrRLIx%QAPihYs74xRqLWe7uB67M zok0)J359w|JCKSsun8hx?r2YNZY*GUwo>v~7e#k99rB~UlDDs)NazOcBX0TS;m18S zBH-a;czG?(G;MfJp%+5Nd{CejVBKmcRGS(YLe)L|RI)S9b2zvqWgp$a+xM?5{7Ls{ z1bE6IwAG8I=PJer=yZ6038@mZE`_~^)B%#5{;%AOrf6IC`~4Sp36Qjuw!on`h;O7( zmG=={EuuwJ{pER(d-=st{#W1yd1puvg9dc|`YfoG;qb<5@8gI!bUc1shhRv6KoSjG< zE(f3vp#d@VP)LpMTXb5YDg>_RBQh&)nNS~%3(xO{ADovXO=c77kg_wIGaXTltE3i=INxg(Nz9~6s$De)~Sm}<~3)mQ{{a=8t-Tl2oH+KI&wyaD-PF>_qZ9#mG~6QtUCmm#vQvf zO8I#pfc&Er3LaID&uB-w&vd%}h+jTpenL|*7$ytY^=}qM}H(U@Y zGYr4!#!nIk8MfX7mkPpqQ zJHM++#*aFK&bH$-6O2K3>ztklw(1S4;O03s+2l%ZNvBkMg`)D*c&`o1cBSo7q888&COTunT3x0b^&bLXon%&AF?fm54 zv;ug-s~Vt8_d)F@CwG}$z-1>DCN2q?KTa6kK~&|CzVSkZ6~&n|uN5Y$*OuIdBFQ$S z3U#Gz;Rp(cX$#C0*?HNSg3l~JiSyik=xWjIW)O|`SZE0de^>RZ__dmtf%eb%UglsQ zg`R3AWh2G^=^VU}Ryg!iO<7St<8I(Ku+-aPF~l+TVRU#5TCd2kwb(AEFwx%2GesoP ze99b`eY(tnO~4`Bv?#`0udnd50y%|%W*ieyZdJ0 znQ~12Kp`}k0u5q1iu23{!?=M=zZV{G9X6MJbN6*SajPMT`-7VAlFr~TUQXr?8?~@v z&u<21U^?^*;F3wANSiOr^~i9^D`Q|~#&aO2CoO}Q4Guz^dTSL}!3TTF98wAs;aQ4L zT3+|xHW44NuUJSj{G60$m|;D1sv~Kc#>)ACWGeb#j%*hYWd(*9kUZ6d$3|I{z#j#M> z0oFjje1WYD6M;HEjJE3FFxM$_*dLFc3Wls~Q!JRZ->qu&&heKyd``N2IjhV8>QUC9 z+(GuD_2xK}Lw3Ek7;YN2;o=pu zzRqdAGR)jJc%Mzsq3IZsFuKW4N7w!0yY_buDO^xt;$3NHZzdD*kFKK&Np&aLNBknV|76wx21X(P7A+kNCdS7lvM}*co+NJC zv-!}XlLE?t|872PGOr~^q@PIKBVPmwLOeh2JS?QC>w}s<^;?UV&pPn?wptJQ4ke#~ zSAy^geH#`FKA9DYvm&bJ;MTN^mh?j`ygN2JkD7=xeY%KiN>q(PGdseNeonGG5;%+o z^<#K43_Jv~O{i0F6vv!0r!26Jdcr0)pQ@#hmdHnQgmkmyAXQ+a$ z^#GYwS|cdaC5cNvL!I`~eK^sIy?4sb4EjM!y8SyV=(->qOx@%jN*AH)K1lRy_B`w= z414XDHrs;bWES;Xn(Blr=g59Krf)V1(Zq*+ocAom%vp2J-yJ*<5-}9^Va1TP`1=9J#uO zzz&*19edzFwi#VTCC{NSdzQZ#?FpME^As@P|4DH^$=wWS**S3ust-Z82L4=?gmcx zH^TbOJDG?TuOzHd?946^BxPE)j`WW?)NnMyxL4v44=*0ks`}u?{~wKgrT(@6qFmvEYaU3S)vjwR K7hk*kdVcrod0wyQkLUJJ&GI=v$9bH`@;=_j@tN?4TB@f{ zv7LfIAg9&tE9pWY$Ach{KP*lh2k+dxo}vN%q4CgFy$dPqxU>L)T!yGA-O=~`usqh{ zZaP#2CAl2EeE<0$>^kS!H`Sl|+{DWj`X6CfVUqf#!(^nS>4}%uMx5mpGg1r*y>GqHW8Qy6wv|h8$~lD*Gb5Ww_Eh zmE{v6Ve$uybx7FxQ!f**MJwm(M}+tI4j%|WAYDGxn|+A1F1aX!+AT&^Kxsom-CSK+ zcWFcA_Bffc2-o7(MQ@98&aOSt)Q-3MG^x)537Lv79l?1ID+2*;kj^k5EIk7Fyop;@ zE(e#-_+MHhn_F3BiW;Ow<*q$x)6{O#_q!P{4Nl-Rb=TvjHREAek%8&$$g;r^h5M)v zlKjz#rs*GWL19>Uaku60ECli$d5)r(RV|YO!)n+>7>me&m6gY?Pw=>{8&5=&WfZfj zrDrK?)AarGYcGG!zWIsuh}Ccp-j#wf7QKsbC>!Z^EYBFTSI!@cQOUg6XCi06Y4SU7 z^K-wwPJUnJri`|eO?Hmu)UR3NYWHo@E(D_Zj;dOcSyH?87g;}Mf&9*laoZ2uqdaOE z;_}S1E#Zm$+DdkC!8bg&D~4nE`+(?SV>Nq&3Q`dh(tYh%4>vD6QD>NGjHTPm=;9Gb zNF*=i_$D=@Rk%5ePEo(9 zOn^>N8i%e~UDbKXA`#VM+lx}bw9_co5h7_{F;cv<;V0``dM>&-`DohRD^RiC*~It! z=$RR%4;I>;yE3wQjs08B%gZauWro2R4|gPNw6gqCg){7d&{rUpIo04?e}V6iJ@I#d z>C_SkYcooMZ59!0GmXcLwjP56ot|wrbacXlkGi`pM{7I;nCGyAvQHfOIzBeEV2HRbl(7q87I zFXNye{2hUn1e4cF5FGrR!O~4PV#M#LpO&^7(on~t^TsR>z zU&Cm6E$jVT{v{Dq04>3Fwk1~%k)894fEH0e+^w$#;a~$wz4WY|L^5FjY#Ns`7PCG5 z^Zsb9->vn4?dp7!8cqgIx$uTebu`=#F1z2U5X1IjbIer{gO zGnlmf9VZZ+Vz#qhwif%&x~m(U&Iab;Te-IJn+&ZZ%g!xF{{c~icAl z4UgMye_3L3idONrdc)KYVcPOl7q{X2zJ-1EO%}EL@@LMR=_F}FAX359tLg25QWJ`5 zWy*3oRVxacJpFEF<$cQpPMQWPsfU&I^HjpdO%NGo08^GdMK1l!yxQ z5(Q^`-QcZR7NNM(W8FlR zp5Vmh69Y=u#KAf1<(+_pJibbaP(MR}$yx;N-YYdJz#u_pfYp`1wKQmW8ssb?+^*%* z7P#BcsgtXJcnQzpB`YBxAO5Tx4&ZG{det4BA1NwfBFibsc6xUYSjQdO60UyXbP%)LuTEfgwWDGKSAVK=wMQ*u)B4K4=;!bIRLja%omx3H) zMhK)qk&>ftq}T2e%+bCpJ(`+toAvTuIXA`L1wwhX?U7p>Y$cMf8~g$JeuElI>C>?y z{nL|^tG*vPu~vSM<(ndW<6drSh?)p;A{@&6Wz7SVEJ3P#l0tPPh#b6nShTjv^D0Fp zeX9~`ZyMW|G&np2v{RRMXdsiL)I@Z=t+gy#N;dBj0<97$Dk+mj%C(Ep9CYWcB#I<>QM%Z~Dun$_B(zWsIlF7$WOi*&}M6m=?*CW#TJ2mhuQ;l&X?H zrbS&(ox0xTtt|#imxD=P*tPT1Z=BPf>7)uaDcO;IVFi7bfq`O8;G}+XQNvb)LV38_@qdHH>v?1<=!d^Sk$|FV?nCKH?cH)JwLY`WSs9ml{=Wh@1Ie+e(EKe;0r7OAdQF%G?3Rz^!IH7-Q#Wal$0>R&<2-Ss^C@)7Sbj-Q~ zKhi8_^S`3@hE;)a61RdI!O}uPQmKZvQM|Gj_$w~!eE!D-UhO{=wzg>?=|)sv3v_fj zNVjr)U$LC{VP5v#7#f`Wcqmmi1P3RFR}>oEtP5mgXXn7NLcKT~Kpur2?$g^-e}NUj ztV;PLsOn{4^cPhD}pXLLhYrs?^%IlG|h4u2|g9J=f9t^FCG-CqD=5rSNdKdEFO|%(!n6 zTas3+hHZy26nVI*^5~6?LAT-ZS?W?ENEp@*83V9_{M*ANy?laa&T%L_>J8L926F1( zax9*qsY2ajg(C}vg(;Fje?0<60)!vJ@U2%N?0PvVU%q@Ow0e~^9E4}}cvtgbA@q4j z$T$_JZsfv^R(sE#d!m)!H#8N8H`zH24uI-R>DpeQkg+Q?FN%gP%+0QA_0m8{H^z02 z0AtNvIs1KIWGIs&zd6;9H*)nj6rAd{`#qt0R9UFL)|FEw6C}9#PHxN$c1)v&2D19B zI`a>peZ8hwOLV2V%P0Ml*sSh(+3Y4uf>7rlkdO&gs#)bkG{yd$HBT!yaM(PbKPBr4 z?Ijc!zvZWaToyRg{zIa`O` zFHptSI21}y?M-4TKNKMVz)0;4`fP7cZ*o+0v`Vqm8}mCL)V`vgcxSt&wpSF`YM$H~ZSzE036d_t$27LuHRC7N04Desx|ElcGl=0)i$SAyX(=)+ zFXMcoyg=6P5;*jlgw|P0Z|fZp?E0ryKZ95V4+B{=V7Evns2Sayq0gpW%mqTP(qYJ9 zE`qJFx;Wvp^-n_WJpv?$aa#o6k4u9`Am2j|b4bW`YXTRDd#5UpHNA|SH>4y`cTrhQ z+tf`*Qa63{a#|;vJi1J%?6z&*ub;Y@c1Izv-`>sF*HczlTdR>YXiSY?x2O`*6uV}X zh&^$#lF3r{wOBGkf7wl>+>UCl^kGO1F`aAYW6%IT01wss66VwGC}HJfm2c(jR=+O~ zjPo5ug?}pYsGT!PV3wXUuZC&ovW@o*HyfgBHI0NjmEE53{3FIgkZw-5DaYoXJc813 z5!d|^HjxQ{e(Pq3bJ|EnVU9lVh)2|ti1jiz_sf{VG*I-2^hvDJ2FN991_@4|U_YJ6HA-yN<@BjO`7MjexTn|74Qi zG^~Ohm-oSsQq$(EXH*41E<`p}_Ljj0F?^kwefC-Uj4o@ovyVXiiy`#QHYI+zmPLUL zkEr30_2IxfmP~y%1C7gRw)9)R-O?eV_vsC@-yAH(+!uAw8t zwW44Wt&-^{OR?81CNv139g)so^65q3uCP8><@B*QNN0& z+~D%j>|b$i*#6BJB%ij0QkKbs+>9v@CoogLdSj5YO#*IDgM_JQO@+GGs7k**`?d~E zD8dxyJsR-!+@=b5Q{Jn4`F#-q44gj8OJX`~2@WOtTUnTMb(q`!FadLyNzSlb@R2>^Bf zik$ByN>=*yEf-j$2!j!3a&_KUINXS4lteQ2e+giADb>9$>VZA`A`j1ICuGlwM{mrL|c9_QASEO!h*B#RIvX{ z_hQN5@Hh?tUyO;Mh`Aa?ZB##h(%|Ij;>lZNKMNq?ih}>SWK$WA z^+=;ROh>`eb%SFNND(z8-dygF&d-s4cm}~wz=$9+x@|w{_f3N2>V~gZ2bI28TW5zR z8k-ih@vf6XJj9i5l0XW3GR~kdfyn7J_iT%fIzNpWkpz@UqOQSiwRzQ=nMw-W1lh%!Fed5Kfd@==>=V`IbZq3}rYAWy67&>LCLCiCQo- z^ZOrxeMlu!ZTtid2W>Q(*gxB6Utm@H@LD5|y;bzKj(YC5{fD{z)1X5C{9$Le1@wYe+-&=(YHV=yCXqsC$YIfa?s>CcV%4B`fP zj3upkaNbtE57e73hiDraib!jGp(>EG_4WGswc_d<3gTlzBQgPmneEokeTzAqsH$pK zXt1Y%`XxZ%^>u^jn5rO7#~^>ukcN zjYkV4V0AwOfyb$K77`Zrd4L_9cc$+ad6pfTmYNjxavKSPrG@KLY$ga^>j~_|Wil)n?`F4 zfS@VWcF5U)+{@QaL;s4)0hM3}+!+vb03lz~)@oL=TmDgH1F+g9P6JShJh|`!dVTg>~QRHE}>D;CDW(J-N=yrw(el z=0oBUSSCYWX1AkCfk~uE4P&2)(3vv`u>K8`f_|4E=PPLl2cv}P=CZ778)`o1Kpv%_ zWq_r>6(c!B1|+9vhgHNUzAbWeKf#Rp{ntEoqKX0DWpeKbWAg8 zaDEAaMq66@FTxf zk_~i(dW2`;vu|uA*nr1Rh{`D_uy4bz{Kys9<5#o%^cvA^DaxhRQ+V9>1!!LM)#;>* zJb0_FBN+3^OeOn7(9gdBZR`Ysd$t+qt!?PbaKTiLLoP49OJJTZLEyoctc@3=pH-CA zaBn=To!dV$Z{tk!eE*#<$1%Vg%LZ<7;m+MurWT|X8=%z!!DoZ!RhI+kNoW9}|5DF7 z*J)7*n3kMu-7m-I4FU0DlFx}yjm*4mc+}HDoE?-V2sDz1MNN3*@k1G|;R0_Z6vxZE zSJn^e=*#_UI1BpoJE{SPA0DndBKCCxW_h@MjWx= z4);c-__y<6tLmCNR+bDboyRJ@Gv;PoI-Q`Mdpidk<=ZP9V&g=I>Lub*(|~V6(!>^d zW!q<)4Zr5qx$98v(vH_dhmz&$I*Y63>oOS%{A_#%)U~imtM_=vkX~}Qb0Kv@!{Z>z zfjCNLe<|+{V3aJp+Fy3R!3Ns2+~x@i5x%{(NjF&&zjKU?fUKw`En`4DX9oppGz<5) zE8#q9TrF3-r5XM2)V0jI69CN#(1D$jQdA5xh5mT=u)@icW+2d-J%Bsbwxd$XMu zfyv${-yfDt5YW#6MSPgASG^zW%(^rmz*=Uf(L*4Leh<-j!{c68`1t6?OYbLT>t`UW zPo6_>^10ol4{Wc~0cRupd|mrbV`{PhuWtUAj{f>KuDt!$$Fyz9Q(#GD*Zq_90@OSK zXp{fwQ2v*ff;GY4|8BGM|G`r4<9O7DuR?5AUCx6v963p5Vl|``B;YUPpUv0MHsDJX z->Ls^tAV=6|6(s!Jl^n7J#s%9sNI$EL-3IXDwhK;qU5TZIiR#KJK}o$Kn2vVl_xqB zo*lCB;Pd}K<+<}cpm+fG2`e2b+AlRJQjp)qI<40cJ+`*YgNszSExm4KyvDZ^Z+BL8 zu6y_sHs8JDS{V%|g$Ccp+t5OeCsNQA3KyJ8FA>|Ft{l2;glJ>!M z9RFm+9x9lH(8R)cux62Mb`nB=K7dj#fB#ByIe0MN@aMM3!ByNT$e)wcGVM=eGRbQA z!grJ(S?ZtfIy!zbqp41|7W?O$r6zVp5WEVAiuf;wt^DuD>?(Q-L(K2LF$xAm3Y0i9 ziV7MJKES)|2VWpVzROT+kMCcl@-f)7gLR`|;l;bbNfvcR77qB!miX1aiHW{^#)Hi| zmGi0{2Y4=r{VC}9K1D&}?+>-q^r&9@jS|?iIZWu0>%r90WX{7dV^~z;wJg)#%fGM_ z|N6EZrC<1`@rSbCrpz*Gtn|QN=C4y7oIbKdI>=p^@%1_Ia^tO!7x>~0K~HiQ0w}Lb zkLp;g%yngGhEI3E*5s%GDdh1-OlXj&A)paqFpj^W8Y{DO(7dX25F|b4Q1OC)C~R<3 zm;GB>jha~)*JTPHKYty+NR?@x^brbz)Bm`pW0Ay2e$D)k%U;GncPR9M|Dn4z++Dq= z07x%IZE8$AuDJB&YSkoBdI@_Cd#+wahA=QBEufEaq}(L;)0p>`VouH8i@~9x=ogup zpAGGGat$+03Y2TQ@1sheC3IpfJw3&nK}SYHnhk@o5(Tu&`-R+64a|`mV^f3$r)-RR zR+9Ji4<)g&v6X=s6Bw#0(fc|qJ!Pzh(!Ky4th_#Zeg3k~hWXv|y^+i~Z{q23T-bc4 zOfB8ZyP#l0SZHWu5dgV~22&Hbo=Qc~$!P}Na-E`qUg4J>5)u+@&*=Fcu zzy?@4dTx-sJV8TAOiWKyhd;~Aq_J*WlJQ`FqHeo5Ha5%zXSvk!6^fSwqZ<7gLpk^d z3n`!)hWit}H6O$SQRsSXC9DDhB*ENArfx2$MXy|*evhyaQ$*7=VSAo5XbU=LpgD61 zV`XKRu%+^{lrCLCHlUcLyNHVTTq`Fy_0MQdSsw&ZU{J!(?4J4*8jE@x2)?1BLEFCAdf$!_@l%+l7$=b#>~JXkHB%o*sYB8t7t8fS~9j(D(}jufL%K2sBQ zHp7gu8!q!qjG&0jFn4Jj-@k(<|Ga<2Iy-0j*DTVw(*1VIv?m!xxs&hOhTrg0*E^v7 zHQl|~OF}8YDF>bjx*rtj=K8mC*LnQ&4fB-?3?m0e@QF|kq2}cceL*4N#%0Ze(NPi7 zMfVXI6rs0rB%DD?5Vv@KrahJOiKn({bP+0rRDz8flmC<2K+E z8S7+y#k(G_EI7ae;Z2J|U{=M^*$F+@@BKwKkQ3O&ML=`(3P*Z8%wJN3N?gDv`xj|% zkthQPGoYwlwe?yxN+6ZB=5QQEYU_eAoU+Y!W7CPKc)rf*UIbpr&wbi5Qc8eiS7yHF z#@$EjAuJ8Aj8H!L9$OjHRe;XA~W1+tqXS-o`G66+FC3r+FIPSKuStVw#+a9=m#IXbJZ|aHWCnj z<@B)QDYA2*mkUesiYl?T@liN~&??ns7~fW?drV==s)(1pJ1WKeKnD$+_9;*~uKN{b z%5{NBpCR}r+b}SOgGo_dXGjKJT`6h*^&T-~SYjJCH9gOy*z#Eh?&`elX2bvuWwXZ4$go z#QE4r$OY`*1XLusKMO(k5*~$!Rt8J~{0hrU_~Wy9jd_`N>m^c^bAdS`Q*sotvlM+^ zvQP+Jz&?=ikQ6;o;lV??co%Tx(pjw?phK`B%L;IH{A3P(qW!w(1w2j{>v40!rPoXuB$!(AlIZ&!Xp7^pWKwBjZT4p`(Op zao;={>APPc@tNon~)<2nt1RUub`) zQgU$uPPB4##vP9IP_3~N_DYnuU%yfb6Hm?1E%LXGE0XZWh43h((986*nX0cG! z{fcq=qpr>-1%_}UrUr~Ub&IBfairAb@GOCZz0}%2k+5@OAmo7QtG=df-!-@p7+R9n zat$VJ2Q5Q6o6$bbCovGNjx#|W#hH|_0;o#YJX47To?_WIky)|u%FR{F3tZFgS6CA0 zke{PUF**eoN7@Gzpuki1ciM?=#E$mLAg={@2>PMGjGF2+(N&17gbP2jBbbh4u&1>S9FX52S=U&B&?o<3^`r@j;b7 z{2SN71v${MC{xM8Q-Qkg0Nvu}CcnShvatD9QlerycyBh@*1v@Aby6LCZ!fniN1qMR zseoh}c*p#g{B?ozr`*656Gms{Ha&f-o+-Y7qJwI`hQ>&?~U0+n)N}l-CpshmT&k`MNe2E?j#_q}h0&^9QCc zhp@5Ty1v`|914c=#M!`5RxN&zlPf%IsWsBg3uG&I^ZUCkSwjn?jm%`7Yn~FuWMBLp zN2fIN47}{gpjY2P&>0OLM6%QaW?dfXf%(mFWe;@yEBykue=s-g%@>0buTwv?>b*dDuaLbwuxMM3c?iyKHY!N&Fs zt)U)- z+L#p}(3mn)XI?5T?MCy>XX4@N(JoxHr1oevo8D}A1^oD4hViA_eV*O6 z5||>3V3ue){|D>N0vO;6IkqXjRtHNrE$^n)7gtLK+joKxR0Shz;vZR&oQg?9M#Ab|e=JivXz5UkBm2>qvS?7o-=hQ@j8B05* zoX+&0P5G-iuG2erGKZqTvIDy=pnWj^wHE2MI2S(Kw4ErdT{|l_FA2zqmGLTOBon>} zjE0hC>u=+eiIUIL=mU`DjG>h7A6%FPI#%VpzCh!NwZfY2OOE~$wr@1@K5&+fUv;mg0=hP=sOU-tMWOoAv<#Ibhi5$)~dkB10PvucHysUXHqB_7zSKPrGV? zR`b&Q)^!k?XBvGYE)YIi=j2$z9XEVo`CzihiZTuxwdmN{v&IkX<*oF>4%eN8>^ z_^{;_=u0NFD}}4pg1sgGNs&f7RSwT8Vvcm24X3Fnm6~v)PpdOt8cdy)1uC&GSi0Nn zKES~8B6nA+umJVvF;`Blniyf27MRqr^D8b{LN8sr@m4q;gPk32CZLgFmG%~C&w$Mq zGRq_^E(O(XJe~m2>ntOsd+rcG^j)^zcVrb}#`G+25Gwne(L2rk$RO={ZIu*wW(uet zy#4u`?C0YF;Rd=2XiUxC(bgK84PzoZPS;LW{py(fqZKpRNV->`+$(q5{@R8YS|Mvw#}z?GB`hp#v(V#hYl8JyTN>lK8I8HNOSeabOZ0Rt%5NXO$I74)@N6gCU}U>>P8E58XX2qdw6UN&#LWp z>==D{ZlMJUG?!fbcp>=AU1Huqx8s7m+nV$#`Vx&>%J??E1Pq4 zbU_fVjCHTlhdO;LsOSJZtyuDH!41W^#!@ zAtWKtY!e*0Mp|AXN%!i9&>mmxZpI4c^lLwC_3x3EiF_+0&l+3M&8Od%LJadjZRwxu zpAqN&(B>)LNN}hc@^J4Q3R|x!6d<*v$GKFytDW1f zZuoGEFIY{Ve?=cCI)=`Wo0~FMvrzdg!zh3WLLcF=OtLFC>6UVPRy#iz-gbRX8j(9m zo9?4E)~3Kk0@F-4w9yRsnI{f47H5a_1#4^14mnq3g83ff(~9LE91DojNe}oC`B~#3 zA3b6?6via87Dz3rJX~vhGByEa8Q!liCMdX4dOK$ba@0 zKYY2Y8}y+!?v^ZN-0gVv`@QEk_Ns2zD<@fR+IW%SlS}5rCEIgv(ggWGNPXG#N0_UE zL8%;bRkMNi)D?%SO%GY9*DcVKfxK$EjVBz%Q%K#vYy2z!UYVOLK3KhH@=soXQtRKV_i$Y%gJ6Z69^FMPCFDFBTz{KnS?V4+KL#Q}b zwE?p5?}J1!0jyu%!UyRb?X;ENF>gr~W22w+?95+i9$%qXdG}gzJ|pta`X4e+czFIe zckbqoBhMJ`3mRXEeA<XgRbpwejAe!!K2A4)rtdOH^Clr93XN+9+Sn+}{@%{ccVa z{v=ID5{Y}`Mqq0XX1`FjA^ozML5(72Hg)TYIbYHL`E?iRN;u^p^r-~R{CPG79tuU0 zbV!k;0E*-sg;dF<5;0xZJ4_aZoFYP)TdX^SVBj_h)BkasPXut=Lt016%@CuydN2P2 z3t69kKB#cL$Se9A=%XavSJRXJa0SX z3RT*JD{R?pDCAPtb-_&)U5apa{}S%sUyBFb=Lc(wQ>Rk?%Qe-Kxzb-!wlCg6A(7Jc zK)2!(g%awe(suvzYg_Vbil%I-bEgo`Q_;{j6iq##ju?hew#`RT4|NHteBciuS|17b z{*#n{DDY6G|B@&K9|V#z`K5~wXxZT(8#xKSLgD59{z|&J!me9ALjmB)^m1VnQjql1 zhvG4P7$8x%AWqT32QY>JQRD+B+CtsZp>qG&`nT8Mg#Wp?A|{jv(1(BTm$*ILno#C)|9&f-aPL0}(c$nv3;OSZOosBv2#|91!CWt->-eFWoa`{AgdYLBe{32Y z>k}*}Q}_0dKve(!+P~JfIi|p90*v+Kp@}e2zWWaoI1=!0uYpMjCd*L{3<5jkprxMy zIeP!05Doyc zSEKnyA9&y(QWpvL95J(-@wO7Y!x#9@VcC&K_mWpeOF9qnMPpt?|ZY6JhaX(Y<$4G18ubp!&VVerY}u@KvvN&qQST zN>s3R&M{m0epOI;6`E0zgHq<6$vAT+qPVn*bBWzl_3^mc*q9j@+%EKc0>S0xKMX6p zd*w_-)2m${@SK{Di$ZFraC6tg;HYSNRii5t89MGdU=*_WTNUF{vZ)I>ss355!zCTs z!!ty>2i6oZc$b&^0E{Q;j&zKoKixdP`sa*$HhrA4hXbD>7`p)IFWN$E5FdD6IzChC z`>ov>3}SjI6k+w)Zno@kJ(!#P;^YWepUCdmkE|EjejibP4O*8DBExb4SixRNd5Lq- z+TepEmE&C(gGh7}(M3@C@T}QBJ%oWnnen5su=u{{2Ij8SKj|6lXs)W2nMC&i?(We@ z*mR3`r|5#GbT)7E^a~>13^OksLUm0fn3|E;|1A36+e$( zite6z#(MS=k2(?-4is(t$P}SI+g=pzD8W>blQX@y>r?3FsrUxOpu$HI$>A=+OLyR; z$)Gk7Ve}~35-_kSJ5`psmrB+(L@T~ty7yvFR9co*uA(%=WH#M6BEE0acw{ijezV=E z)!XNSrN2E_HZPhxt#^aQwj3OV{5Hu{XD9uV$9g2V{5&mflb(go?;}YT5mO(Zv9e!` zx@%7N*}&VIXzAr72A*8V2vQ++jd%!HXY*RQTZ-SrJ~-mxM-6gVlMUi#Gehv|7?@!S z8$vcETEP5q@(QLSGN3kM0yG!Bt`S00V^=E*55Cg z39O2u#Qq~y%pwOkqkhRwN8%^lTvm>YG4FE;tekAs&5S+paV42(;_qaEwAQ9Q6d2tc z8!4_TX9OEB{#G8ec)P$bzqtAvbE|-Zs!%#GyZa-J`@anG`x@RYFFI_k+~$o_LpK68 zUHgGjaN6G0!ry-NpjjZS%X?Ag?jp&p{Jl1E5^Lh+&|=7MY7n^Y@!E$ zzGm*aAmvDOY;5lDU?+Z$44cQ{!r!-?#HB8IHfhv6)s{t!E05@H4;JYNk2HEruGc24 ze-nsR&$6_16z6SL>2##&%PcTmmPJr!Kg={`PP%$c<9pH5z zY9{KeSK)F%eN}Eg`=Kz?gZ&rQo#1X-@#~_|AFdu83N#v7y6tj9*;!g;AiOp8~5L6MH-nCSmXsn`!LtM$>e0$=bt%BiNkIuU=c`wj)h@;~q*ZR4Dyr zlkuZKJvv*tcSil0SVNyEdWL4OLm9uJl}F}g3Lbb)Tq-kaI%|B5FDCbC!LM>XM6)7( z#$#VgJji6llg{>fzodkA;8<%-2y=pYdC2jMx*{Xe>KWm&mG`M(E5f)`Z;!Q%(vl#S z%J=GgslEB5T;Cnt*rkJ}4e|%TgDTr+N33IV8Xm}X^XkNyC}Nk7c|H`pwR}wcrZ{Vj zoQWcf*F(XkwMG9g4#d^F@@X=sfQ|I;8z+o*yqX$39PP@p3Mo1M2b0$^2Vty#W&fQ?5nBcbH-T7N7 z3SmzyEr~UtvsEgd5Xvo@$R`jH=t!AvW)ml{g{}Xias7It?{U2 zp=gPcPWGqsWC z59mI-S0DSmZYgqED1G31f&lPC#{8vWIdSRRXdl zS5%by98++5;^R^`W&bO#QxaEDS0oAC31iBzg8nh>-e&&kn0ObIq66GiTZ7qR%&PJ>WfY6{2})ru&a%h@^2WdYI;rQ;(~ zO~;CNEw{JFYF>b`)z~fhVF^HxvO5!rM6;pJ#pO)rWVVZ(j+Heakm@mI3bCDTFZZ(i zVejetS(*D|{LS~8;PLL}Hf0k<_T=Jnw`!mQowysraAsm*1Z)jBHV-8rX`}d%g&zrGH)YRL z!eWJFe%ZX5PY2QlWvWmv?hXY1Xq_ra7vvmO;Y1S+98EE_EF@-F&Vl`O>*=(&fX=qJ z)I^1Xz$1Ms$o%-L7e|d)&bS8Jm-#O**ADAdVGW7#!lFVB_pfgKJqrNN89e#|4is_Z zIfaz@T^d_}Gh#A)pwK<1;=jw>rMR&*YF{5#gno>&0j-OQ(tygh&-QD7T8pVV;GJFY zsp}T5dcT)8Utz3#jIKC)?fyBoAUtuzBx44_vw`D67{Qvm;U1uzN4s&j z6tjBmG7JdwMOo!o+=(l>v1(LXXD3u__7GRK5j4~~o=UQFkQht=0V`)TZx z%A!>sGB?L!<>m+a&mDF&edeVBhQd877dc`U)Nq{UE>C=~F((*uO|PmqMYlWgjX3@3 z6)%9oi}?6fLE-dR5N!rc-|AJ(Cp?=M(kb;@LZ2nInZ9mGdtQgvfu@@!&p zm|;1|c0Y}pTT?`-gjQt>m(Ffo-jP24<$=X+r}s(hE>n#+UAVa)q<%d|9<@cADuVMB z9eE~{u8TaEKP*?BS^un=j&VS;+|u4%@SD>Oo8ARO!vS_@x*%TxkX;C*mi5myQpFZ9 z^YGZIXFs3H%}NfHAe&x|ROIlPh@X42@|14AftF|mQl2#nvy~0j)8S+BvEM{Xk@Vz& zBf|>Untu&fc{4olu(4jeMhXv0G&7EPw!cME9L}}h^luZX_1Ds46vYI-m6RF>V8l2A zxeb!&*47rrj=eOb+-J+u%M6vn4ZDZo;{t=OH{}o3lEb8`dk)-k*`PkRiB&BTW&VQi zERKl`Ir8#0Cslu{p3D)VU;-$6IDthd9k~tgiE&KNMWJ*@3HHo}qoOxuJuxPV;2BJb z3Li@YR4A`nmaeGWZMWY$l*ib=mY%D~fqNrcC{So=(4MQJ3T3q#k?^+ek*c0BWmX*b zJ~#7)Qh~SBkeiS7w=g#6OQUG*M90&(()t3OUSU_4wnh{JglERdKW)dP0JQO0$FU32{tTZuuzG zOYUjjpICP{e23;J%AO)2kI*smuUXHP?zmRT}pSxHX zRKeux^N-gBS5UO#KXimYK>)19>anh&8aIsQpx}vfLq~?eRl&x zx}!~hJ-Evcog~IPpjM6%N`LXfvaLg;nf#+C+JVNbd`wL27NZsLqL(mXcH&AKc!;AD zc(f0`yLX5*UoTguE!nk9*1dGzLMRh%#uGXkwm z_{@;@p#VHnAc;s*-W=?E&~Wd?R?=jFS?MuQ%$@5!Qe}(cOHpt73_Pi4q?d^alc}mv z+qbky+99lyNDaHOWuA-CI{fjNWOOGx8=F{foQ|t-qpAP)9rD*>r57pK7Y^_}YROe3 zow+QeG(gJzL3CPJ7@;_JKG0tP*g6+i2CN8;rHv;!8%gx=7C1-Y?1gHtOB@%*kJT_O z1V`D9y+TNjxgp*_{>VnTAQahS=iOv~_ie zeCA~)a$bbSf|WwwgZ&idmFAvmbR{!n+tORdMaVhcSvS& zA!}Z_u%gtLIo8=*t^2iz2xCjDdL2Sz>48|q!titxjdgd;m2P}i!Zi5oj zMCzunJwPR#3ds(YF4q3j!@_ggecI+pDkIn5sS^(NrV#A?m)f6}5cc;*+*;jlg&XAR zW2WM;U>Ygd(}JHXXli$&`^ltwO5YLbLcspduRe^Tzg2O}^Y;oUW^1hK4*0SxmdC^j z#u_e+fVmh2GA2y|);`f^5JpSH^H>j^)DmFUAhgF}b!5E>O1yF57CTeKWn4qNbpA2V zQJ^0AkDZ}vr zEvr-UOr75zd^vYC%OjC zvAmlu5vjAQ6iIGxe6i~TQ;JojJft+=vaYuqvNsuruzKSARXH=G_Y$@6If#0yF1FxF zf4U}sZvN|72_twM&88v%#7}EVL7=xcZ~z+J`@_l5gPo-?T85;3t2)3eq{_QgPnW&c zYC^wsJTGp`WP^l7t7h5XYq*l4g%;VHCT3vyUrcw(N0K^x#>u3*T_47}{Usyfy4u@J z=C0`;o!xz}Z$p|pk*j75N#YAsQxNo9ugr5Z1NLphPEDzELU4tp!LEiTPr6kb=%}rXk0eZ89lC)sP z_yNt)7+{QzgRbY-j-)Sbt=-%G(X;5l*9Wckkp$LdSN}6zr^G~YSZ-7wglodK5rg(v znii zBKmL#Z5mCGNtvpQ#3JX0DHl$;XueJ@3#HfxS<~L~OquUXA)k+J*>5Ha7+ICM?^Mes zv^Ep#c5zFOuFp(0l(5?z%uRsj@ah6KF_(*ldylYPWQ*#as{e^)FDg_RU%9;`=Jc00 zLyK;HAF%h@@TS>#Jp}OA#8g&?ot{uw8{j1=wAEI}crKdaAq?`{QdOp3b;Blej^c1Q zG??ciM?)o0r?vqv7QSCwd+oe?Y+dsylbmZt{2+~41=xsya*Az>a9cR8rI8OB%nIQ6 zfb3nViT*9EHX*^LAJc2^rydwxfr3maDn7_EGJ*;XAbt=Xy1*zZD#6yP2Y@k8Pcs-a z+vpWx+v<>6p+M;QrSS1o({6)nC!QB70vhgx<#kYg0VWaq`+|YU?D{sTo^K+ic0ARA z^U4SOQ##v)#+zsP#uis10EalgguJkn47jA)?yo+5YVS3rc_|NH*>m~tt$6eV?`Sgy zV_d4Y31|fO${4EWilCqnx>Mm`CmKP_(i46AK-Gem!z@?-NxQ|Zu{(0p>4u1C^;;78 zWm3KkUmj|A7Jz$0>w0g#O<0?08LG$B?GG>p;=|{b`3C9ID4rYeLQQYktbnR ztRJ%nhHXd3!wYaJlAL<4LXS_!Ar$O~K7>r>+%w28^Icosc1ZZ*mvUDgdYj$2uEeB( znYPLDw>`FQnYURLQy)Mq#PhI^_p~y1ts`F!j>!djH7I*g3CEk+d}t1r`_;O+mnLL` z*w*PZut83scD#GV1vkt4S)X4=FuoJUzmz2-rbigB8F@F!O;De8Zxa!0S{}6oC<$t5 zu4G~1(x&;Kv5^F@j>{JGz^ZYAoD3p=OI#SrrwGnU`AwP41~ed!gH{HtxiSReVoGx( zeGDVQrjN--TK6ne2IIFji=+u;zs3|9aG}M1Ygq6qL!pc8=!sq0CJIppB8T2Ck3}=U z36<}pB)|9fd&3EvnI5JNjEqB$5^LMAPq&_pKgxy4mpGAh!R!yiIEq~_4l{5gPm+o3 zSqWeud~d!5X~gCp2(aaJCc{MCmKXHnMd9Hi(psp{WJenx$y(B)4wx0K@V{1l1Ju<} z`ATSo$-}HrAAo|pG*oe^3*qy)?v<${`whWiq%w*fcQKy=~t$&q^T8y2I#oldKR(09kw90g`v(NL-!k&*cl`{luZ>yYV3)c z)!udn_p1Q!Oq%PC6C0U?rGf5(k9GDmAVXtYO84&qf}hci6(UsdrFrdFQcFH_W#8BZ ztIW{k;*)S&+CKYRpg6VMIvqeWgbMX80Z%Qqij1rJjDOs29$8qzfgg>F1qBu(sf+mx z@rX$TQ0Rbg=jg-$doMu4(G%M#VRd^IAOfvp8N~+bD(1n!EC6-QJ-P4)W)(43}ZCO~&Z4+8BL zff;4+Q(l4vUz&dXGI{$8E2(Rn3ij7*p8L=6pY1 z^!E0)@(vPxXJ}~UYkMP`uPngUWp|erqyoO*fJ zvdC96r&ksK5f~Eqy`d;Xj8gr#&!xqi(e1$WBFdtpS)m-@*vH7b!BE0d-ToZf#-`I5 z273YKl0jMv1KtDV<)?6PeRM(zW2a}Q>>h~uC`8dVC_R=}R+?rS6TKOnT&z3<`X$RVVAoZ%bfXs| zJ2SaLeB#uVpY*(MAa*(OetYk@u`x=a0hLMv%=xb1S9sLGqv-D!S_~O5KS241i7eve z{J+?H@1Ur%?_U&k)Dd(T6hTR?ppr2Fl7osgf`A~Bp%qY)B+03p(Lu?G2q*|B2#5#_ zl0$=mBuLJwk(|LMbT`lqcO6i_^LzKcS9M?AdUbErD~y z&Hdj3;4SGUWj9w97<|8-0NVW&#WOc%uA%-RGb^pPeh+Ga?P=bk<%`1EvA(r=@L<*p zd9bT7+2iZGjPL*W?FcT<>$k^33Ts*!Xu65L3Kg5{dhAxlH&hn0Hs@~aJ}lM+t)`Au zaDeewa(B-_?{bkuMyuH~I>tzDr%H6icUD8WodLB8XVq8MD{z#(Pwv_Y4;Nwuhb3^i z9XvavUm#DSlkTZl(_?}}ph7QZN4Jr^_CX4i2fE>OAUp#pz|Plg3md$V){W%i`GtYE zIfEsKi9?T=K7Y={l7=3Cd*pW0TK?D$7%@7!8d|YJSIL)6w@m{}+6$ex9Zj;$ptb0Q zN+&9GNXtGrRV)g2j_tjjb;56}-F_Ax%Y4I-of0re8we;auGj3BFkh@Nso2{P zsZp_d_9NqCx~-{5_hh6HJM*0pkxZx*iH1QfSNIbHrP98wvMP`u2Q{M}J8pm3FBpUu z|1%f-3L!W%lKbuD;olS#KixKhymlxdk*=MtjJID5u~1EG+e-4WqjCyI32J5P0-NH0 zHQRuUVIWza6)Bk@aO^;ee90|BABcrg2vw=y|8%O`4+Njy50sAGUj!Lv_Pvtr~zdRD7+8Fz^YA7+w~ zDb2~v^`7s#n9ao30tBd(*N|^>yvtzVlW&HYsc?UJxXcN>$A|lDVQ(PYOiY zeD*Cnxz(MC485QXumo8mL<`fe951Hp*Z|g_e+aicqG=??MmQ1EQ2ljTX+L9n#3kO| z?tZ|^7{DJkl4I8B?vt@9!v9ltFs?6a<7zAvB6YV+P0G$LM67RA)7bDM1S;jNO5u!i2`-cS5J6ZLXzYZ5}qoUMt27!k-& z)%rC%|6g5*QjMp9{Ll&(HFCQTmqB>*KeI+5H5XacF?K7+&!s=2s23{1;xQ?#_YC=H zNpnW1#&42?u>-C&2|c+4V?CwUe{43WrUWQZKL=brWw0rquA_#FqO%hJX(#*yNlS@n z>Z-!vwAKlXJc`)2DL+h#|J*k-!?fA7$q81>w#HRkAnNOxx#Vze>~eCzASG0HlVL}m z)PL@gSHMEl+fdCwUl8q>d4zj%KAT*zNgmo9Ut+2t*=BMwC>p^Aw9hgeGK&2KmlZZG zVW%qaVs?2_u~Nv5U4S`9?{^+#QU{TZn9k9^eC=%q&&xx!QQl0h9g6J8kGD*m-}&6q zzG)`9Jj5#g7en{6OM0@Ybwku^cr0ZimBvXY9rn{ADKgMLqF^UlNPT`>JClVtX|0s* zL&2lbJn($(_dVmjo3DSn(?D5w*tlbx=qFygUZwN#|0fNv64<1PJMJMSCDd*?}8D zeO*f^dibS&$nXVzFa;TpRt!__6%UcXWPn*Hv=Xzdzat7UGvXVAkObn-mchZU>05Zv zb#C`^^Bv&e&FsuY&r1}VS-NL;d@g>`o>8Hj(a}vH1X?KTWV>gS%Iw**X6E5|Od0)< zC-RD^)$TGpRCCrhFbNKA$m|AFXR|V{{ymL5$ut~Z3fd!0o{5Cbpyrz*X8#|{2yVGd zDsbEdo5J>Z`JG4b8H|F@LGZ~lx$2D6@G73uOibyYzkG?aaKI^$NMv*DEq(~+%5aT;i4*Q}3Jf1R~fd`l2lKnOng)ZzKQ}o)Uy`tc9;`AfAv(JLYe>gyCN{;F zC`8ty$;~>W-eOm?w58(%RV=wVS8G|g>E(7YF=-J7Luo7iZ+8U>nAP)eKkw~+8ZYPu&KN1anUvQ$Bj0E`MKiAs|Ggh=76_*pjCQ4rA= z)+uFZhD?H6Y>uW8A85*lZIlx-P%n|Sxd8RZ=p>rc5`B*b*Jm6cr<{8}$rT%tRX)TL z8Wa#N?}5GCnOBn_nBw1!**vlF7G%4D3wLaYP(sWt&si}WX>OF% zlEYraIywTWB5ONjm8B^i9|rMxk*kUAYd61p#Rr4y1+_$Cnl^{eJKlRRgutdIMh-1j zS$gj7dPsb_R(9y3bZgzT*WHC#*gCtC?hUJc0opsv6V+gG?(A84oBJ;Vx1@gWdryI^ z%7I*&uv0UkL0O>V==z=?G-k%52$iNg6R_aYhsv`yT8hMFkG34IKuM?yaN&W0n=K~hTKRwcovnms*%M)6e<*|896V8A=&rJ$2 z0|`QS4N@8TwQpEipL6UmXi6N7Lo|wIA^_gs*INc$FE=3`V^JYJmL*1SxUH$&hNN#F zLVI8a4ntYT=c4n2vR-VPU5Tp^S29kmDgfj!DalvhbwpTHqQYf^H9sOWeU3tU1TO!P z6JLQpkU2&1hzF3YyLUdMS&$o+!UWM0w;CgT=lzBNENBXDvk8UHe31PZM|COaxKyNz zD#M+gCnVUfoH=;g)lRN_ZQSXzZ=rH(QU)&XGS6^Q$r_p`vZ01F9GSd6&JvweSjcOy zsafD?@j*>tqsLN9sJyo=4G0=EDXOhoddm6UV@q9|Vg)>l{w z9xRQW(1e|$-^k=)d3m|#25Sq@_U8@d{w{F5CgkQa*%sJY=H?N)a-f1W9;TI>PP1zi zQf*v%c}mGv-dzm5MB>y0MxtMDlKtRX|FL2t+M|Yd=FsSzrr#gaAyY}xvfknx@gq&< zVj?@(;i?A^k;VDChOPs;3-H&}4cdoX6f2cZn56qrv*TmNNV~L!R8)rUQty;xM{L(0 zX`C48`;E=~^-oO17n0*^Y|~nRlZLw1VfL1#_LrM#uQH%3`zG0r1YcapxV?X7#R~z| zj&AOD?0}L~U^m@eWTK?^GlBhPKv&(qo!di!+3+xx)=wSo_|lx${A?#yzKX_{aO?__znWRd&42v_skq_hVR(6Oj| zbFv2tPY;TZmcg#R>08YfjESY_Pis2RAu7!|hM#-W@y7!M zjqNVaWe39~%yrZy)qzXF)??u)kYGTT2y-ww0B0`<%$?bMm8l09*K#k4WH~uLJa+v7 zvXkI%e0RS0S7<_lQACcXXI$<(RcEmc8K`#3UcG1dd&JKioC*`@+;4I}vUHa8i?@dSrHmtmfO%0iI zP);7k$0a&a`0>bv9TJ$$z=_;+BthbbL-DsXUv${{v@pqmKZnL;dcJ{-z^+B*<Z%2CI?{cPrN#&0_4qlx@wY|8 zvgi2XJjGXf+D9^e`JES|G_Go=fYO53)ok$}9TEJH;9qxSuptef z|NGJtyEoWIyKmF&;T7M?D7`J|g$-Vj36?+m2)yq9Sv27PykDrMH2#5uwHo|0VuWL4 zPHZO~(0w}x7w-T67yb|E676&e1&9O%q!ym zj`L^)By*+})4^wd40)Cxpk(%zH5nH3TI;NcFaabRy)wHkk>9h4!;5+6}Vk1To zba4>F6qU73-*WjI?7qIKNqB7JUj^6Fbs&H^39f7=7rlf^v34NH)A*uQ;gkCNg7Tvo zw(OZ)%4mp5!}CKMbQvF<;%8AiD^tN7^)|3@WO3DJgfo9}rs6Ss1{RV>8L;1aIdJ$J zZ#%Bn-*;My3KK7m-EK^@+@_j_j6txmNnZfS&?ct^Dj|6Va#-|vS01M8yhR8kc$QYy zi!6yW(S)WulB*?r(RD;0`?H9Wz{BboWKfLgy6Xii*Yx#59S~>@M9A_h&&-c!M@l(e zH8C^@PKnB&oGU)e1Bymj$>yX8VnI-q_B$xVF}-$2CnHVaTS}Ys{szazXC+Wm8oc$I zrQ#Gwg9&Jf_{jI-#R*;_67IVCib52oA?LQ}(G$2>TpfNlsuiSqFm_MbaAZw?-w#nz zJ#S?t`PQxylvPviefBR{HICfc#6A)PD9?|eLLq3qPDXJstD6m;LAt$QV{P9|%y!O> z>ByU2Js{~H?D9?9z|X9d(5*a_IxXaDZDU4dLszMvm69j*j?6dCn?ica^UCrIjj{h= zr*z%jA>hZrZ@Uz64p9G8sYTIpYGx+%8Ai4QUz~5pl@WUP?wnZv{LGBq zJR<^ng9qEx{7sSC%2VTYD}(@*(E>eZd5YQB@Nj_7D$>>mxQ_VOv5c}rOI!LI3%?N z1b4P%Y~}-H+16GA_t?Dfg}B+CjL49t0@K$ zh`{kH{X?N%a;n}&T9dA~bYHOPVZc!BBrSUK~8 zBNO|0tK+Dv)G|*EwY@uqm%+VPQo=ZB`b;D<4kPk=D#-5}PQ#f%Q@8h$Fjvi*K#zlpu7iCN<7yT$vrZaLb1Q&a{FB69$VWn?FH|R&- z_aIt7j|IY84jktfNQ^-_g}9r9w+EQj zQo7~b2n5KcDxvou-}o~Wb?hTSnj*Efr)M_s%4~|hp@E!uph*TkwsL+Rv?i~h56%Vp zc8K)Oe}Pw*l4mJQQ`g6&ef(G%2d z@*8Il8-$&9ZX_tAk=I$$*^Xp-Z5Je-rWwSwA)9V6FxFu z%dXwnz`{s#FspSni9|!`>&nMw{HT{HDID$&gK&P>A1pb%X?M(SS+7=}b$4MZ%}tiu zv+kXomR5h({R_eJBODYad8%4S9}@+o@WDU;yXzjdbNT{32t`zPm_&f^q&_1%IAwtl zIsQ_=7kHh|K`CjI^V1>siaAP4_tv=IwiG0t*Gn!o;TEWW>n)R0x^Zf?Z@J#{K#}34 zUArFngT7VodM=TOH{uTqIB`{@p+oM7n9EUrSAW#Yzf6k0JGaMOo)y(rgwii&w$i5R zMwR9J7vkn~ZpN2K?rtwk^JBhtI?;vE+B5O;Y?t?rsMMgn!pGOwU;7<&c|30r>R;8? zr*(xa7B;fG-8s!~%`3SOP0x1)==!r2#|E5_+7BEFPDCll_~47h4$09gJTJ(XyzZXB z#?BdLmh-IF<68+ECub0ElPSmiKuLKHvir0QbVMxHQm!EooX_h9qnvbp`w+~<_1j2` zPa0CBsOTA%oGz?Er998><@$#t8foX+O_p~{Lmu=hJ60C2kejWhS#n~c&F?MdF*j?2 zSozA1lZrBLXV+gcH|uDzS1|xAxi|CAn8vvIqk$V6rd6$E7iTjeQw@z85=jFv2KPd9 zTTl63uf5l~j-t#QY=zw<^!~uG#>EbAnC#o7aw%Ouq_ohXIy^jqqnte4;o`GM9N=Tg z)V!^3hpWhvvr($dj2mg`_u1DqN3Wo`tMuuoCM~?2NlRbbVMYOWbDKM)cD$b{_5CYf z2M724@FJ`XovSIK!(-kRW>bz=L&BAeGL{bNr9pA|n>fsV61i%A-n1+>Ht@^TQT~^| zS)-#cy?mzST~i65Zh4cSD`aOENYtuo4HjUgC$mLfWwWB=72Qs9 zGxH^R4tTrUgS=v@)uHJx_h&~?fiaQSbBgmhnV1ml6bi{xR4$x%PQX74$;5Cy2?(&V zw+}Ot;1!t&=BLsfn61kaQcE%OrML0|F+AaY0KNhhtR|Slx7+0{+4LVS_K5i}XY80G zmrC@dD~dR5ZD)9U={P&O`Z!0DdCtm;y*%(|x?oJ)$ow3;(*!A0U*A|-ztU7z1~PbX z&2@F$Zg}Uym_fWBv9-*YR9_#Q$<#TXA8;w^a@%*A@0E zM&5x-rp(*~ncd4aZ;$xO?X5s(FL9IB^OC*rH#0eJlv^cJhTPoT&F5pxrixqT%z3oR zL!t#L(-;+k5C;r&Edfx;gU$B3X0r3L2D93Rf`i5SdcJ<5FH1gNq9WJG$Y{7L5jUK0 zfK5@)672&(pU)Ca^U;EOQzIJe_p86knbFmrLc-bA8RXosnOtlTEMdhANnntytyPHrTL0xaFg1a3 zBRunm+1p2#`){$4@>_;p6`einRgBRe+G}<5fU|#3+rVMDfxY}Mr=ItCh>Nqc$Hno= z+arghX*5=jw%-NHrt&66Gq12P<;-yX?T@ibt;)pLc)Tj2vi$VKjKAyBaqc~9>t^44 zwG|_)CTAKP=v8C{8fPx`@r_IEUdB!(AbKrbA-vhxJt^!`6)%yD@jcWr{ZV`no6nXJ z8J=W5c_+_BI#K)yK63>|MYoF}%sspW_<9%zp&oMkD!h@jaWm|#sqc9IsxhQqXK5jC zl8#WSq;wdkDW!&lCc**W=KZKVXc#f$oxY_V%?eY1b6pnl?pFx+V z%5IfqGQY^{^o$JAjZuk;pux?d((WE#=NI;0@c{$*DOPjrLF1i`%xIO{?t%p|mger} z`SUdGoFf^*26lyV>~IazK4N{vbbCi_U=#5ec(3W4M`tDcEY?`Iin zOt7+_2{DtBA)^g+GiR#^As zPz&or!GioN{rQ6eGsG&lbsQZqiEn9t8b3%gSn-;>>*^~P9n+Px7(EjW=CUf@=#ZWS zAF}(1y9`a_lT~+c_B5Bbu;w%khSl8{M~Q^nF%Q1>jyAWmyOhJ+_+6g=C3sVyRsE_& zuTt`;KDz6C5HM>XTQ<8;!pmCM&_Lv^Smj1eA91@;oY>=iPd0qr5imvl@u?Bba9?s`a_*g=Eg5FM zXU(rOrRPab#tYB)motHA*~aei1wfFAX`2*ORW57xa7X*{5WxnW=}@*_^0dCKy`33w zX*lmk>1kV$o)%>XoBlYxLN=$6Svahv7KAtblK%Xe^*83nZaQL5gz~le2L~bbW<~$; z{9j5@^$qaT&?4WN6>id9Sa9(XJ8gphD#sWam=G7V7&^s*-oc`(aM8qF4LKkfeMcW5 z2O%MCxK&lwlQf*U`&gC++S|Lo*p~-LuU0O4#|sS&DU?aVT#>IccS42F;nTc>rgXzL zW&D)>fO}F^wpe7A+%5pGYUc86c+>NpUSP;*dXIN{4Z7(-D00@~W2|6%EzB2QNL`d> z9)U^z{Wu|ZXKX^~++3lsnFCU2w(Lc0=wE^=JFL-{lWj-O2SMG|vlag^2iIF3Rx>j9 zxhq+XMkf_Zu@>%80zOj{np_Ef3xvH@i*+{dvK^}7)9h@*-5a%4;t~>QBU?3>yu!V4%16-~(SgtmR03#?B=X{g}8o^RM^C4lN}>G`1b^>u(L*0{htgAo48 zsBPObLnEW4{QQ&9=`#eJ3EkTcjnZD#COr(!y~x8bAYhQXw&b#R`49h#N@}Y&&mJr? zjg5_kavU;V-X$}H!3-TUOG{o7iQs!6E)`79vjpE_aMhR>IsoYYg6TnJZ5buzns=;)k+RPe<>BNpJePMVa_gzl3=ySkRUI4p*8! zz23-8rcw!58iRtn+rf-oLKOvMavt`KfQ?pSDk186U!w~7d+*HsYsu2s@b<1gT3GmR zRzeLa+F~(mWcS(DMb&|sd&NCcB*dET=`5-UJ*X{kc%+(Q)qez+8z3iB=*W#-e+8YT zM&1vF7*s9|T71-s=}~sV;S^w~%x_)EVP@-rNhVWhkLa49>6KI&ZAHkMfO-c-jKS5w}Y@zZ@sL8eM~ei?%h%iwE2U0hG|hDmTY;w~2D9n2ISJ2JZvteSt1 zblZf+xE?O)S3 zbz!X`Gq>5#g>h9+;IO%wkJzb3tK=h0;3~RD%>Om}R&z%~_2}}~^sUcBB@hZv*e2^| zRP|F^(>J4C80i%0kpUlX{LVU|zJrr|YC!a$9m~7o)UX1r7YUy3^plcPT(#j3n46XQ zi^ZL6=qHQ1yvcxOpojUe)PIlviFa} z!LYApGrXy}t_U$>tK4IZWr`ojvG*S!8U6FnyfJd8v=S;p+@) zf-UVj0Yk4YOmRtmZrM4#JnI9pX=Su`sSBeq4}2rT))q8)W)>dsphicr-C}YwC{W0I zm6RS<+d;=xW#FG!u^JZu#d^7es@gW6s#@Z{dTzWR4@zw~aalbw)}~0Mbp>~SSm$R6 zhUAndq%(d`@urn~2%5#l5Mflealt7cHSsqnt8)(7cQ{yG4gBA{roUcgR#~Z3iT7sO zq$Z}(Hd!nlegdhNMb850#xa{SAKPMUHkFL2v=bYlZG?)TY+}{|1KqbvaGx)rrTM1G zLY>&;AhO~jNPNc*F@uQtSjGmmi3dwA%|q{?bEJ)?e2+Z2xrPzk_#h>9X$PGm3vCP| zutrDsI;B)*j>%YabFLyqZfEUp{-Cw0jZM?!*;vbN`EYL-8>0LTqf@(>kn^BHMr?v1Jn6>nxbMO|tYhJzXy@J4r#mZmL5 zTWDHy9+2NOVU*^lekkSxNs3~p3e`9aw|bff5-R>N5T*NmE2!#$g7McxngOD0bJ� zBz!a8>ImCL0V=ftQk-&eq;d1`P0z0NG@6O_0;Z~q@;KXfM7a`;^6hTgMdvr#ZnLl< z6Nr*z-R^~4dIkSaadBo6H82!kth-7MDmaSVfm1m~L(O1x3{Dbvv7Um8ZFC0C36+eF zbF_JPcyk3V`H&;*YVaame^)XZKie9}1_OW8x88O4&G~PPUsiJ!WV$aN@Kb{yv!*J+ z4LwI59y#Vzv2Id{*|^=COjoq$1oqpfuW+bFEn6mmPMg?;<0LLy!QT4w);TDIy@0{7YWB6pTuB~ zLZxX-xR>AGSEwKS`6J*gU4qYt14QW0Ao<7Qn;U~^FuFy*rOA!L`kP2X#B~PF`!Wg4 zzQ?>bW}CRnS5FNmZnOn7Hs+ys(e0BGCMI{VzX;ZKK!!i&8fA4nj*)|bwY4LP^m}ay zlAMUy6r9rh5+`s8BtE*Vv&eg#1Tk`LtXl8LqYUy`jCDBZ-9D;7o1P;RNh0Cz zuOYZ4hgL3_uMZ+9e=%GK%EIqm@!z@*#_^4f59ACs#vQmf<|@>^#pvh`C=c%y`HCDT z`&eAhn6jDy2Tja59HQ~UnWS?_PdrtDb60I=VZankkr%&$OeBKG!9`VYCXrO3hRn>o zM^6OvdHfq_0gxPZ+3UfVSJC&RKrWoDFO(|BEZG4`W{|?9JeRloCZT?D(Pm&~r9pV2 z`!{gBA>k}EE2|Oi;0yEQ*7iby^67f&NeUB3q*)s;XTZsh$oyi{@b)&6rB0f{b3<+N+45;LE0N;c>UH3%%KB^E@%7o% zaB*{aEf>DFxdcbg%LTAB2mpZ$H*24>GU0;$7nfiMzJ0|9C7q zE#EWPF`i)pmNG2kWcaTl*A=QF_m5_`@)CCiUfX&^1k$PDC$Ptu5F=+1idZhs6TdER z_4%*M@c1d1XG%ZKF}xu<%lSCd_Pna0n|_FZ+E^YhSxw$G*gt?o8!|A!>_zr?Stw>i_kDV0z5+u56YEp(qhJS1p z4yiy-8_R(KBUrro4!L#guynU7lEMFq^W#a6m7}O5N5>< z?VRghw*=y?oU$@DNPNi2d(SpA!+yZ%nQVP7Yos;fP`{yJ9n{P?gvSp6=9=l@sR8jf z1UigiU1J^kCMLl)4_Hn$of8+0A^q71z|3U_&8{J-g4_(z_e(}nCUCQyynBveCZ-0V zfG_o4S~~`L`p~*)(wW8-z{5&PrQueD`N^qy)y-Fa917ol)4Q^@Ku#qyG@$L^bd09( zM8v!7y;K|xr>B^kn~R#NYoL`74zwR&BxoRi_`9>9H$v=b*^3vAcVF?=Be`~7ZX&6c zj4Yd_rP3oqHRPm>T%NHA@l+a(<{>4;shI$^wNJ(|Oc=m{Wm`ZL*~r#b&Bul828rvh zW#3neL!_vnfQ{{Wr11V7oPWX6n;@^C^4mbMIFkf#b*ms9GRL##ubRlIlE&0Yzam{pQ|vZ_^84LRSbM~I&L_;aV8A%0m5P?g$Rq)WQk9y@7agP}z;gYF^~M%@k@3iz57bjN zP)u(CD+YrKEj-$myF|=r;!?7#SuuUlPw0VW(1?|$JlU?UZc3VZmVg*417e?Qc3rG5 z_!J0-#RWQ$?V)>MAQ)lSOiUECeBu(G2N$na0mel;!d6DG1}Qnq;O4Sq>R`*kOjtZ# zB{Ipt-l$fYF{@p!vAt-M-+Q{o77|+w-(_akZ0kllefwwbkHs5F3=`tx_(KbvWWCQG z1r?$9%wT@r)8w=a(GqEyq+%V}GAe=nnUxxGv`#W_njZ zebH0Oj}H52>xf9ygc2j53R`$gq^RrE$CR-s1W9Ax>PFr82a`kt2hh&$p$`^!v z9SC?})&oTnAVN(QI|swDpOJJKhjG8!)Cg5D<39Cwk~FrKBh<4r4Z&3HI(}5W;8Jo# zDER8{zSJHdVwI>veJBlP-xBG*k=#e*!@g{_pdx^dj<<4f$neJ5+z~0?n6G)N?7EU5 zkMm5LybcZIMGZoE&K;K%_q25Ma3%V<@UD56&JZv_3lHK}<^s7@OgZzQ=*9fHod7dp z28KzYXOVAf+lQ^Y+gCOaTO_^q^uP#jVbcA*8g_T}tSv1iHa0RGpfT0ho(#2?w6jlx zs*28@Kgc*g2pk}`hq9i?RWsZFX^e6BU1hkg!*(cPp{`7?prGIdb(mJz&y&e+H8dHC z{4X|&Dcby8{iTNoOja_D?@X|0Fkf1<0!suL|5{{IBO_bcGMtQI9;b~YoQ_~W$N_by zK9-P?^F8$`MZcowi1VHqFGoB`}bIpeLNz|mAV$%mA3i?SIkf?!mSi!g%?0!#3`(?tQ1X)~M z3u-`IoUGG&93K%1l|M2ATS~|*G0}s?);?T_>iJQ2MQ~ljYfWcoel9`GrRrPLs&w5jfc- zq&LDp6}$uh5HFn*hH6*d<`!=%=JA%NPeCCo;S?s5tVF1{k;yan(UkRW{n$7&y2>-Z zu&`>t=#cAzaQ3_L^9E8N*OXC)IIQ z`n1+Tg~z{dN1l5sj$@Z81}5vT3Ob{UyuF8?Hoa`&5IQj!5Y5Bz1yaTn$#!#-)X5@~{X=qy)?CwHUX&lBT;Q z6z1)~lV?{my6X+<3!$%In6kYcpy2P?3Ey*LEAltckv2=gYB74oG2r8EIcXC7ELemU z@^ag#y+-du?);l%e^f_^Wu&ozSV9KcwxpjVqW8`)Bfh)K(#%Kc9(VFQfi2rxz!z#s zj*n;G1iJe_06vc3^vuv&f>k`7*$G4GYzOPcq2ppcOl(ElU>i+jHFNc&$$!n9>#ztMB zEZ}RgSPRIw@*tI!`wlb&YBAoo=&t>KbD-Sac82||s06Og(&tzpO66}POfhJC8T<_5 zH*lG}C?()9RV(kg&cSpAcABtMo?s!^v9;1i&9f6@BHLWE9n?cop^Dx)$!0yhy996} z{CTRL2zhVZdpo}}sY2rx>S-f%veEDkK@IR0L>rq^RZJ%>?|Geni)Q)mT_TO_%T-KH zT2+O{U3edHz*@`w_FzBkd#|+RlBHg72ncr|$aW7Jb-4xJ6^Mxg7o$9?otWs#l9>f` z5h0Ey*4t;847+?4Q^en(#Q4062D4PiV$&mYZlR_%C$H)I%}xHq3%VQ*iBXT1Y?Ugh zIi3=u9u59~l_!+QeTTjK3GRr}@ZN*Bkhq5MT8CmFS)kw1Hn0*Fe!|u(oF){$`7YNA zn`gesjbfOpuNQQk3e3vV5YD}ql$n`tkI4xQZTPy70?r+*qq*;{;J0LH zSSQLt<{EEfP9Wxx*yJva$Ku<;!~u^>#_&bYv084;^ds?{b}%;#S_e!jYF|A1)I?ku zqFiL@adq7A%&jfU&1Hu?UjDGj;V{;0FK_8VVuXbjM$>m_)x+?VE!deEQs+HvL(B*x z>4d4Vf*=tBq8|}do1ZrolHfXp7dbGM+AT}>JSH5GlcddE|S6G;@TwJ{R zH2jopSXz5$IJGR-%9o}-^fvvPB#{w0XjD~C4{cIfSlN_q zX)F$s{?;3~KzL%Dam;adWK5Ln95=|xrI+l%Vd3)2EmMKyKwr=i& zAr%m)oMK2UZiVvSC2?~jE0l#tq=M+D%Aamq2{0!?iO41~;!;K>kPOH{KDKcvtvy4NPAw2WJJ(`NU~lR$zkp zNeFCzOsH-owNm8j&xzb=LA`PlqE12NFazFYm^fVlCs+>DYJeuNbLf9`^K z*AV~n^A`TVWxcZJN55&59v5Mj-M5cU@%+y?1j1UCL}Zk<01PPz)4_oJ-$mD>fk-nD zgz-mgE;WN7BDLH^HaBj9QQ=&`*8h?9OH1lt|Cg;SuBed|9Mi7fJU1Qme4$MV10yNyr$?2Kc! zTRn;(1X->_+d!>m|9pi{kAMFz}^VX^2<>;B=H%5ny6p1 zss-k?->(7q*<0}|ev!5&KTz%MpLAFUCSLrQFY+StU(`?FuTnEI<*wxE zTm%5Cj(q%Y0B1Ci-3NnP~KEdklqm{4iR5so~G$Rn5 zbvngRM=O?WIII%Qda7P{<+5O4B2{Wt%*%}-imB!hb68hdx~{EjX6}0*Y^bEv20o}=lfWUz znE&*Ue1$54pt8VRi&Yk)au|)msq|NFa~LN8hfvZbeAM*WUD%uTg}BNIy}cSvV#W_c z8`cu76IfaWqk^6qHjYkF*___Tr`q24=>aK6{J_BHuE0bL`*&&9_bP~8Df)**Dkur{ z)zj_L<~VSr55BY#D!~w$DJ;y(LVBNCnz8h~!X3&Mds~tbe+1O$)23t>_Y|=w794V> zJTJ_b?zm=dc+5$YmKtLtYi7J)bk2cxqU&UP*k;-3t-Wt+-HLsSd<=L7sG?BiS-Q5M zWqV$yuR^G^p>a1ggFA?dKTuP#mtkrZ030*tCG}VDOEW~50uz%bx&i`jGfWk~_Zg)+ zQ_0pWD}Y%}sl07n6~~0JE*`8nW%XgkxR(1zNP|Bvj~!8{D0uB753UIKl*+@0`;0JB%;N%LygF z*|8ve3&%U$5esWi#F>2xDgESJA(mi(LE~pxk#T;uwZ0FONXkKCs5+WnQ>{KDC)rnb zJTudwGv5>7+@wON#GT8{T{pIvs?i&Mqv} zEPwS0Nn()d6v#wB1jM*ykLTOhPSs&t_UZQ(sCjjV0C+qkWM^#x$3pqZDm-99#RIl@?soW zm)W_s?kz}nYsmBMcnOGdL0XFhw>{VwvV#y|UPgQOo%G4M-e@iL9OZa;d$sHlp^C+* zBC6ODGRSY2b-ec2+1tlkU2xeX%za&ubLxK(KFzReiJ_j9RQnRIdXh`Q`$i$36aXo9 zQdic?+$Km3Rsh9g8~}U{)@uW`H)9Tp3Gm_|yz1C@d>E+!YOZNGT#LC0CslZr4Cd|0 z)U_p2rt06sM^C;yuCCq$(xdUAnbg$OB6o7~Vk~I_K6%GiSo|>1AF!S25DaoQ847CHD3Kz;q$-&M#5CZx% z$+z`@NRx5ZOzE=8_gEhFAJ$X`fYzYv$rgcl-9=qp5?19ssV^=~mDH{`Mz2)NnAkgVhTAJ(baOiYO z#{}aH$-XZ6w7f4p&KM@@F}f>C7j*t?XmALM@{Ur4LV|5jv;Y*%FYaV(2ZD}~O|6E+ zv_1;D#@R`a8q2ZIaMH_6P!!YN%+INJ#;>z6S*Any3Y5rMDLFmz288SLhGm=*V%@kT zeJRnxBjE|O+^(^Vh^`vzpiH&omb}(f0;Ti(1nJ6hGO!z)pW0daW%g1+%x3 zOH=@SGa+xd>MMmys_1S^=RR_+stX91>RCNx$984VH07XzoEJ-Q4T%v!Ei_U6n$|jc zGj>&VFb60@6KOf6dYHx2-=^3@?N z$Ohu4OL+$XV*6JgsP7SZsA2(#qLJ`&Ds_>ygS^4HHOTea@fA=Bc#uYsgqKiG zN#eX43&0uzMwhQoeF^majA%sB7%Ue6H8Bd0+YIq0k*hne@Ws=PgZ;Ynr5o&*{Xjq% zi9{J+Pn_=chf0T_{WAoR9POZ~=i=_FtX3>l6=Awn$E2iy3+O8c;c~$G5xhAGPV*2> zjCa^Y^W74nVmDXZUG4TsNE8l#$<>2JhQ>I6RM1rk*zX`7ygZa)g!|Xj^%_E|OWBIK zd^wI6Y#)AQVeuUvFTC*uMU&q6KF65jazg!mp$NNfSedjBRLrTGr6MKfm4WVaadlO4 zaGwa|mW@Hm(&*|kW_<}`ZRgO1co?kZPa%>x!D6Zp`&F)M3+kW?M#+XsXxR6#>bbWU+GCt|6o-N+hP zT++-MPbci@Xldcv91Snu!sd0uQhX4Bzn7al2p5udfDiLF%6nR`B_e@VE9UUu6aSzm z2UY4Fh0q+}y8tJa-se1B1Sc^7%kf<)=$kZulqxYCOY@|DZyNzV>^j*U%A<{s$3U$D zevoRx2y!N1jOrR%+AVI;k|1{-p=%rG?LmzNMdCZtQMk={M1D#QQto<&jD&P=-!LHP ziIh?}feDhRLMNQQufNG0YgooI1oP zLm~Sh9Ekk*3DVmvEzwjI4yyQUn4N>M00EVST1s3?O1T$v%L&5++8!`V3_5Ia%4`XDh6x00&{-_*DYchA8 z8+9tKs6ZiS^|Fxk)jliYFhGXR7|uR-;{v-0CF>3W;2PeuTDXv0P+WXCE-sC=wTL60 za}ExEXn?e@inY@TEi=YeVn_`6A&UQSFmeGlZ5GYMiV(f)qONm;MP9&R!ThX(92~7y z-82nYcfdRy;XE3o7ug2oh%&lmP6}6z2`W|A)?S0tBt(IdZlqFXQh}~nTgM)T!=m8G zkKpcZozW{sP##9M$Xx~w)oJBY{cMe>%H_TPM5@zHe#t$wUgx;d+iM63EpcaK$2YKP zCaeO#+>Ft0S89QE@&T0y@)Y1g&77=HLS>e7REq^HhtL(vOP0WElUI}>uW*^?6s*jk$T0Wo2g8Y*;TvDCHFtC;@YT6t2QKIEYCY)k)SjE8-B*{#vy% z6nv`$RVl1f<@~qJ*q&Im6gb#Qitp13$#|yV`9w!g3l|1Uv_@vrr`CHyw0mC@I-{Rj zh>43|)^zV0BKG!lJm2W}YF(jclKr8$*br1~@4j3BxjU4-gR-NpnM#oRkp$HX!3_kZ z!E|9{SIP0hX1GoX3uMPR^E|7N$abj#>Vt zHOWj}#<0FN5=o^ET;wtzaaYP3Fi^>m1%W7fbGn(UFku~#<#74S7sk$3?J#7-QJi4` z0XF$=9`Cq}4~g9wx`z}Ug3W4Z29i&XtdDtHiK(mmm)PFdPTvu(hs4H$NR%AQaTrTT zEiRUd75&GurseVybb&8^fIsYE-c!)r-<6b?x}lH;TH_>roomJRs#U|Mg*of38OL;L zjM`n@3S??bFYUdYfA1;#x}%(ythJq;Q<*79-cEeFyOmjf40KRG35^%L@Wi6jEo5aY z+DBlAe`!msn;S;J0lWC~3!{NsaDV{P*nfdlNXUxZ{f8W8ON@dn5%_!E1c322eZnSD zi2z&HTfkQTmfyJ@AOm&vBjT+-aQ4UZ&p?n5e=3Q4U;Kmzx5O*|!4S43Grxd~&Mn|# zTL%NL0)JCWeuw^t_x*u#{Rh-IgtW{8z!y|k@qcUXx}%!R+C1Qg?1wX=(nY~Gg22FF zqZdWNHXt?9zX1e>7DEXo5aKwabg(d3AVfMMVIUw92#f;(L}>~LN)jbVm4whj31sj4 z283_Vw|jQ?>>rzRJV)O8+^65)^St+73lsoE1%?176mkfCA4&v8!9XvVJu+ZX!;NMc z&m;(InI5gMM}i(9_>wh%@3XM-I)h-WYJ+OHgKGN!WW;Om0hn7n2kl6LCOq9q7>|7N zmKaCq=DrF=0N)R_ITWEmPo*}LV@*S)3Y@y$_mh&J^knHb1fqp0R{)|; zgfSZd!UpidxJ{i;NT3;75O3B74&h3I(*m3+xc3BvY(CngENud0bF%_~zOG%6IE5kx zAs`6C6&nQ;MHr@oUm}nS32 zoX9jUn3)g-`LMe{<1oDA8PUd)NAvB2U*FlgyzHGTM7XX~?}`ID3^kKMU1kOQ?>%}X z!N-6vquPa{i}7>|t9?uN3g8%o=fz6^aD z!{wp&y6J>r^FHLk0+93T>0TJEss)qw#!by++UrkW(-}xq zn@lywVAHq3Az)zp?*DB)7Dz}z*zoC;CjVylc8W#FeJS(DnV-~{5LE?Nw_O- z6WLJLZObM-g`mg(>Ftjo#QUrMK-_GB4nb{lFytkG*f$hW8Foex?Ef!~LmjFMz)l5n zjgWeP-#!Y2)TfYZpAOxA9$vE65o{C)F8JePMjFI8+es;`x zNpz|@z@mXZ<|)N>H4+jUnT6VS|81I`lX7sigFn$W!Jo}Ku$sl!r*J&`N5Gn(&3rjd zLCnMXb07DB=EOLw!=S@c9cB8BP{vzr(1cwaBVAUR2Au^Uzd)ccQT!MT_l*J~QZbyj zpSoP4QEL-amCJf&i2lcr$m;|B;dcX!UU;G%ydIU*T#rigFK_XlI{c@Q{H5Vxc~Aeo zYXb)o(#afa&)ZVE+wSc8PUfZRSgIZS6I*Qdw0-rg_(vzb5raA70x_@>r}2tSJwn*! zTCsLZA`M`~N3=juBXV}ajf)ZO6s1adHo@yFe|(dI?Pg008Cb4|mY zWiyI}!hex6PL;m_oWi*{u!HW6<$0$DC7gM|!u8jQvlt)<6A707{;m5Y8CV65b_}y$ zW#t-nk&yiz5&{>&oX1j7SG|unFM~gGiG@w zf0{w(g9PHBYyB5Sq&0^Yw&E%rbr>Qu5cq<7&*3sqE_kUNg!x>6p3!o!xmrLOknqw( z-C%PGo#2NC&NH`B8%EjJ&!3BvgaXSs&Xx!AV_d)toZ>eN9}Bt$C}D9CsHmu=7o`eq z6gkCq{>K0xAC>su`mve^B{7&~)Y3dY;gW8bp25;bP;)Pl+a0-Ew$sZy-@|jrp+23{ z?|93+D^Q7xkxv<9X1&YFL3b*S~V)<>lk} z3*N>Tmj;KxdVPGwl7@m{t7(t=MH+jV@~b)6OlVa{X87WYYm@SFJ%9cYXcu8|k1pkx zSME(KDVX8LEK(D~SEj|F3oxWIdH(zaV;&abCTDDeY=;?B1mbxn>R}=LNPuRxz zCQGhHB|U%MreOJsgEBKFK_obV_F$6>TSKO1%BYLh8u+3Y80CV3GMj)|jJ!nl587b= ziZS}Lgl3EVVtPaQU9||TL!fs@BYX-RSZ%3cxsoVc9v-C_x(d=sR`>Yo!gJNW_ zDo9bIB%yK<8F_`RBTM#zIRP2szLY&9r#CD?C^dUIUa2bb2C*N)bMGwILCMk)>ve1S z&H`#QKKRufAR|aMPY&LHY%53$Qoj$$dlW+2U<*}&EppwjSP#kr!iF|TlKT25#Dj>zUqN0Xbp*Tb7;LX+;I3}n z_TSdbTQC6fggw$%X*6yxq(7V=L1B1$l*n{$e%>j$Mg|p-0aGiwgLE90>KBmso{NJNJk=ahb;tqaHiBmW zU765%r=>9q{qgnvPLxw#&@C;JjGLf^{A}Rd)(3(SZ8*k0@a)uG5M#db0Qf~h{nxQ< z3~k+;13^G8^?!F~$U79#!+8|;UF)P5gJu1-1p))Sje!^eE9fpNB#Qt2yZ);?zxn5v z4A~>sWVay<0S#YS18EERjcWd9QvVNkDqF;_n-`=q7g*4|a^Kkvxov1cGNgC{>s&uQ z6n7T{AL|eVhbbo;2|XGpb0Q=rNST4n;|2V$g1+5I*&J*)vrHKp1FY+tm3xCP|DC%> zi7!$%BnIndPEFuk{=)TtNdho)$i7C2gUw&{QJ_OCz4p*`Ij@ZcvcgR^2Ac%L3kS=+%x@aD2-r>hOEv2FYOI_5!6;EvN4M!W*sx7548W}d z?`EY`oiFK1bj*?et-Ak;imZDeI^TF3le9-E>r5p*Hq-H#h4Jp8Z@)7v;$53Wekk$5 z^7-zbQAZCC4}18NcW4d$;^W)i(ViXM89N_%g084EhtcgPKVw)}GMbQB?k%vqMp96b z+_ul|iHQai%)tfQHX5amcv4Sdx_l|o?rU(`;@Bm%Ettb*l*UZeXJ%%aSJD%GDe^XV zCChZ&;@wdiH#470RFFudLTymUEs(ioUWE{){xxursOm&ImMNwW9_s^k_30b+xp(om zEB7^Stxudlepiuu^~B zbh=uKbj&8&aX-x$VpnghFJ)&z0aCxt3#4{vxBF10y5|M>W)MI7b7A=pZt+>X(TN2b zo8O7L(y3<<&xont8FQtycDac?)EKU0-x|OoV(5=DGGgCp*2e}Np>4a%WO{)0m3iiz zM~qdYhcy^&EPr9>kgAiuRsZ`q_n%GaY{nC)rB|q zTxDBg=ie^0A!C%l5_H;FHt&o2drYMdy9W|sllO>6l$q%9b3lHB;iZb^m>vxBZ56p( zicyPQJg0_{GhSM=#rjOtH~?gW{tRf)DcvOaWV zqBZYy)QsIt<5I{9EbTySdif`P{CoGS5nA@Y%jVF1pdC&9*|tc7k&!Q74A4*J`igoz ze1i_iv{F^l#8nAKy88MFj?voa@gyCC+7)}#hK7c+ip$BBJ)d(gW=)6(lVDDu!o#H) zrHJhEY8pq^Sjmmxtdc&EUufCh(UDVK6NAzUVt;HOA>51qExeArCt=&?M&u0or>H1( z?2=L)sFAIx_NqN9bL}60?rr(u7V7<~kz>!C+@tTQjRR2z3MV|->2GuFE4WX(jQS{* zEU+~*9v?rV_NuJB_I^S~`lnBy7KV${IX%Ui?!h&?0qR?|yNFmKS(D9xZic+`(k#^Dp9~ zJDtDn4)dMUpTIwkRY{BIM1|^tnTCY+Wdtygs9(TjO2R$F^H5!QnT%WQm4rwS?x4uk zX7SN4gR7j%re*O$9odGu{s%?$as1*HCzsYAX>XH0J$`(}!7(u{kE*XmNJ#KhGTP>Y zauvN-$(22*P*xukMzhh7b`SW@IMlKVs3vdG&#EaVYzqb-LBovbi?2+iFE~SY{N!0hPgALt=nFDq_ zgU20VEx*7y*P-i!1$V7c8W*V5cs0UrL0b8T4WIIYE4HV&*Epu!Bp3K8*_+79svwJT zz;Mf|U6QK%#}c;PQzdk~aiWo=IuPl>PTY_QGw{m9$;=O`1jmXnhVeEZ|NI@Msszh? z));M0yf8F8T-UR{Mr5~wLQ;fur?2|2B;U~O-+sX_RDcgd?;X-xs1C{zA=jEj z%y(%U7!z^Buc#`;I5OIbd)%X-H?|??gVLc@+X;N2i>=AFvhs?~v!WGBZL}3R;26QK zdZOK1LM_ool_I;dGE;7Ana&4hK664xEjf$t5@o3ky3Fmq<760Dm8+=W3{>CJ|vWXLgM4A7Uf? zBXz;wK{O;`hD6e#mr}a|4e9s+SFVAAV)A#PmX@|A{WV6H86S+F&xM-qw9=6E2*{HN zwN!@Pk{QTmOTn{}hFLX`09doOT=n+m_pL!v+ceCuPm-lOd(jygg*bB8B(){>71PU}aR*tRunWAtz;Q(sn)tTSL3#AJtd(VmhiRM zi7LSYaH!12?+k_hrJUsuIc<6iC?D>brsid|jc{9fg4RW2wSC|tZX2iF%Ptl;7UMZ5iIGr;tQyA2=gR#Z$*`%JwO zfe;QbyBh2y+I^}6j3G$Bbv*Sza%T&`3fkHEDd2oI+kua&$Bm~WHS4TPG9GO$HriWio$!a;TNZYQt=tQEEU z(RqXkrPr{-@LUvO)>e3O(ma@*E#jym>rrQ$*ws~G@s9D%aIWtIlD7CBWKY0BxSs-S z28s)U#4D!&`coVo*yS|`;Ur605A`nb(V;@=NyAXfC;5k!H3KUn9E3{{Cv&=LOGMts zwc4K;6ITfnn1HRL`5k~0bsN3_K5Ig{vuPMz+uvd-+U>j7)u5)JteEvO4zOUZ)vH?} zQ;kr(?OyP)w``5e+o!Lcq1`pkIe3smO{iQLq^?MJcDa6^OjIR2uug1~5hII?SF;>k zl0m>RcivA~$`JQ?Y|vNr{!0n#2z90>@4jUoYphXA!?RT|Da~Ejau7W1#z-;v2Xp}n zJTze378Z@Ci{w7P`Dkl0jxYFhLws~}bXw%IeUNl#&l0-Zu=-uFs4YXT1s-xUNM^#U zxCU-6$gcZ~8x9zM#910LARyTIeJ(c{Px48ByOIouZvo-1v1$yku9v$>U)Bumc;9OMw1k)&R=i)24BQ=c-eQg=?&Ci_ z=wK541lQg{H^bBuSrNbJ*_ouZP3nwY7CsTi&91JI^~lN5a$~uPhS(jl^J^=x;)V>$ z$sWB5P4k9Y2?!G;1An%p%Y{a&)lu>r$2%*GKNE?MrBPP;qKCu_xnyq;PJ_`}cO+^{ zajrBM;k`;tr{;dF7+26b>5|AXe}c0}-VQQ6B5VHvqMb=;eInSv1Ed=9W;4U7D+@zR z@xmP>(lH5FVM;0HVc7QW=H_NPP2-p>#Id#hGd9!HTxK#)C7 z4o?l&>b37SoXxk*NC26Pg@vW28ll8zFMUsWmZDPAm>m@A;`-aAJIA_CRd9V#QGY0m zj4K}Py$GdAacz?|Y6SH1f;)l88p8t&5!T%!sNyOyFi$co`$55&H*RNpvWnSe_PJer z6iRH<#mg?MHuguHDWS*uu6X^1qRr0Q)=uq19ld5J9K8Zk*;_c@wwR!0~GtbDhf+15lYIccm&cP_;dZ+`h~y&oXweMKkS`5))W$_L!*qC#t4 zPn)iWzR6y=eo20nWduotoJ20Txc->jGn|=G5O~1+I-JzC=wQ-u^OncYKkvsE zV@63~AP86i^KXj$JWRsT4QrrusTEH%<%7vQkUTY|4l6y8HVHJ*thgaH8ttVA^uNb& zwx@e&sK#430Es}B;56_w(jGsxi{tn}0J^L(J41?V_fz0W@nCkAXoXA3*)xrAPCL*V z{(O*eKep60THcL9F2t!m`oNi={GcM^)noWg#_ij;L16NY;tYaV3fx-`T0cMkux4O- z5$h)$SPOwxh?}36a2HH~JnEdsxx3y5yK$KI$aM$zqQeAaZ>PkGs{78puoL~;k>^iv z3zw2n`)$nfV7zY@pz=qr!U$fJ0zl^k{SUGF7nzWi)_(hhSK=sG4n$Oh!G1!MoI5 zFkFSSolHF<@P!aAI6W+=TPYu0aRpR%-Z1(Wi(Pyr76JM-ram8Txexh;p_f}Sq++-< zY798IZ=K9Fa5`zjWUawRHgpy2KATM>aKJ;>oXd5Kc>bWK(HMWL(4mi({0fE|4zqrB z?$C8S2Na6<`;gm&X8T{I?~Iv4thkB}n1VqIm}Jm-p3@Pd$SHO^%UjGcfBib@4IpyM zP;u#jRTldIsIFp5|3)SNlS>j525%ZMKcpHjo^Lef2jBDq4mJ>a8zha?T@bW5)D}(M z03;y-K+ud1W@r%s-fby>$l(=gtav8*YgI!j!06oow{3-yRv6C*OZj^G<}#ormB?$1 zk0Ui~Bi`KH-JulA`!y?+Tal9lily=DB|mmY)>w3H3U{ny`J4BS*IztdcMkZd<$!tw zXmF96ePP>bT3ImVZA9fpJgvmO?vz=}_p8|e7Bt}!`8DH=D{{{9>tzX)mADYD0BFWq zg+WRj6fT8?3^b?`vyCu~yjcJcLWhWXdDv2+I|2lF@l}1|0#$Q~0+yf&2TLpOECq#G z=~O9LGuo;cW;~{`8IL!z;tC2u22W!aZpFsMVM|5s91X#;#{K{-tWA-a0M$YwL*dtV z+n1IU$188t)w#Ork4|6;|7Qn2yjtchH1CXAuG$HtB}9QBTj(ByUBkbwlq7TjyAv({WwKzjO{f}S%9mEVZPyhe` From cc16e348dbd603972c5155425c0b8c50051a515d Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 6 Dec 2022 18:58:36 +0100 Subject: [PATCH 77/82] improve description of discovery configuration variable --- .../network-dashboard/deploy-cloud-function/README.md | 2 +- .../network-dashboard/deploy-cloud-function/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index 6ac87398cd..f0fc4b5b9d 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -54,7 +54,7 @@ tf output -raw troubleshooting_payload | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [discovery_config](variables.tf#L38) | Discovery configuration. | object({…}) | ✓ | | +| [discovery_config](variables.tf#L38) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empy, every project under the discovery root node will be monitored. | object({…}) | ✓ | | | [project_id](variables.tf#L84) | Project id where the Cloud Function will be deployed. | string | ✓ | | | [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | | [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | object({…}) | | {} | diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf index 6de0eb5d93..50e17f5794 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf @@ -36,7 +36,7 @@ variable "cloud_function_config" { } variable "discovery_config" { - description = "Discovery configuration." + description = "Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empy, every project under the discovery root node will be monitored." type = object({ discovery_root = string monitored_folders = list(string) From b7fae9ec20b4f2f4a1933c7dd2509a128e65fa6a Mon Sep 17 00:00:00 2001 From: Ludo Date: Tue, 6 Dec 2022 19:03:09 +0100 Subject: [PATCH 78/82] add comment in example for custom quotas file --- .../network-dashboard/deploy-cloud-function/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index f0fc4b5b9d..a7bbb24431 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -30,7 +30,9 @@ discovery_config = { discovery_root = "organizations/1234567890" monitored_folders = ["3456789012", "7890123456"] monitored_projects = [] - custom_quota_file = "../src/custom-quotas.yaml" + # if you have custom quota not returned by the API, compile a file and set + # its pat here; format is described in ../src/custom-quotas.sample + # custom_quota_file = "../src/custom-quotas.yaml" } grant_discovery_iam_roles = true project_create_config = { From 395d42e74e6550c20cfb07736945dba3b8161e41 Mon Sep 17 00:00:00 2001 From: Ludo Date: Fri, 9 Dec 2022 19:24:10 +0100 Subject: [PATCH 79/82] rename op_project to monitoring_project --- .../deploy-cloud-function/main.tf | 8 +++--- .../network-dashboard/src/README.md | 2 +- .../network-dashboard/src/main.py | 26 +++++++++---------- .../src/tools/remove-descriptors.py | 8 +++--- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf index f30d7f2a2c..887289663d 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf @@ -94,10 +94,10 @@ resource "google_cloud_scheduler_job" "default" { attributes = {} topic_name = module.pubsub.topic.id data = base64encode(jsonencode({ - discovery_root = var.discovery_config.discovery_root - folders = var.discovery_config.monitored_folders - projects = var.discovery_config.monitored_projects - op_project = module.project.project_id + discovery_root = var.discovery_config.discovery_root + folders = var.discovery_config.monitored_folders + projects = var.discovery_config.monitored_projects + monitoring_project = module.project.project_id custom_quota = ( var.discovery_config.custom_quota_file == null ? { networks = {}, projects = {} } diff --git a/blueprints/cloud-operations/network-dashboard/src/README.md b/blueprints/cloud-operations/network-dashboard/src/README.md index 4f9597f2db..27dd159c33 100644 --- a/blueprints/cloud-operations/network-dashboard/src/README.md +++ b/blueprints/cloud-operations/network-dashboard/src/README.md @@ -20,7 +20,7 @@ Usage: main.py [OPTIONS] Options: -dr, --discovery-root TEXT Root node for asset discovery, organizations/nnn or folders/nnn. [required] - -op, --op-project TEXT GCP monitoring project where metrics will be + -op, --monitoring-project TEXT GCP monitoring project where metrics will be stored. [required] -p, --project TEXT GCP project id, can be specified multiple times. diff --git a/blueprints/cloud-operations/network-dashboard/src/main.py b/blueprints/cloud-operations/network-dashboard/src/main.py index 5fea7f664b..6db262a669 100755 --- a/blueprints/cloud-operations/network-dashboard/src/main.py +++ b/blueprints/cloud-operations/network-dashboard/src/main.py @@ -84,20 +84,20 @@ def do_discovery(resources): {k: len(v) for k, v in resources.items() if not isinstance(v, str)})) -def do_init(resources, discovery_root, op_project, folders=None, projects=None, +def do_init(resources, discovery_root, monitoring_project, folders=None, projects=None, custom_quota=None): '''Calls init plugins to configure keys in the shared resource map. Args: discovery_root: root node for discovery from configuration. - op_project: monitoring project id id from configuration. + monitoring_project: monitoring project id id from configuration. folders: list of folder ids for resource discovery from configuration. projects: list of project ids for resource discovery from configuration. ''' LOGGER.info(f'init start') folders = [str(f) for f in folders or []] resources['config:discovery_root'] = discovery_root - resources['config:monitoring_project'] = op_project + resources['config:monitoring_project'] = monitoring_project resources['config:folders'] = folders resources['config:projects'] = projects or [] resources['config:custom_quota'] = custom_quota or {} @@ -222,12 +222,12 @@ def main_cf_pubsub(event, context): except (binascii.Error, json.JSONDecodeError) as e: raise SystemExit(f'Invalid payload: e.args[0].') discovery_root = payload.get('discovery_root') - op_project = payload.get('op_project') + monitoring_project = payload.get('monitoring_project') if not discovery_root: LOGGER.critical('no discovery roo project specified') LOGGER.info(payload) raise SystemExit(f'Invalid options') - if not op_project: + if not monitoring_project: LOGGER.critical('no monitoring project specified') LOGGER.info(payload) raise SystemExit(f'Invalid options') @@ -239,20 +239,20 @@ def main_cf_pubsub(event, context): projects = payload.get('projects', []) resources = {} timeseries = [] - do_init(resources, discovery_root, op_project, folders, projects, + do_init(resources, discovery_root, monitoring_project, folders, projects, custom_quota) do_discovery(resources) do_timeseries_calc(resources, descriptors, timeseries) - do_timeseries_descriptors(op_project, resources['metric-descriptors'], + do_timeseries_descriptors(monitoring_project, resources['metric-descriptors'], descriptors) - do_timeseries(op_project, timeseries, descriptors) + do_timeseries(monitoring_project, timeseries, descriptors) @click.command() @click.option( '--discovery-root', '-dr', required=True, help='Root node for asset discovery, organizations/nnn or folders/nnn.') -@click.option('--op-project', '-op', required=True, type=str, +@click.option('--monitoring-project', '-mon', required=True, type=str, help='GCP monitoring project where metrics will be stored.') @click.option('--project', '-p', type=str, multiple=True, help='GCP project id, can be specified multiple times.') @@ -266,7 +266,7 @@ def main_cf_pubsub(event, context): help='Load JSON resources from file, skips init and discovery.') @click.option('--debug-plugin', help='Run only core and specified timeseries plugin.') -def main(discovery_root, op_project, project=None, folder=None, +def main(discovery_root, monitoring_project, project=None, folder=None, custom_quota_file=None, dump_file=None, load_file=None, debug_plugin=None): 'CLI entry point.' @@ -285,15 +285,15 @@ def main(discovery_root, op_project, project=None, folder=None, custom_quota = yaml.load(custom_quota_file, Loader=yaml.Loader) except yaml.YAMLError as e: raise SystemExit(f'Error decoding custom quota file: {e.args[0]}') - do_init(resources, discovery_root, op_project, folder, project, + do_init(resources, discovery_root, monitoring_project, folder, project, custom_quota) do_discovery(resources) if dump_file: json.dump(resources, dump_file, indent=2) do_timeseries_calc(resources, descriptors, timeseries, debug_plugin) - do_timeseries_descriptors(op_project, resources['metric-descriptors'], + do_timeseries_descriptors(monitoring_project, resources['metric-descriptors'], descriptors) - do_timeseries(op_project, timeseries, descriptors) + do_timeseries(monitoring_project, timeseries, descriptors) if __name__ == '__main__': diff --git a/blueprints/cloud-operations/network-dashboard/src/tools/remove-descriptors.py b/blueprints/cloud-operations/network-dashboard/src/tools/remove-descriptors.py index c0645d91cb..93b1110e46 100755 --- a/blueprints/cloud-operations/network-dashboard/src/tools/remove-descriptors.py +++ b/blueprints/cloud-operations/network-dashboard/src/tools/remove-descriptors.py @@ -49,14 +49,14 @@ def fetch(url, delete=False): @click.command() -@click.option('--op-project', '-op', required=True, type=str, +@click.option('--monitoring-project', '-op', required=True, type=str, help='GCP monitoring project where metrics will be stored.') -def main(op_project): +def main(monitoring_project): 'Module entry point.' # if not click.confirm('Do you want to continue?'): # raise SystemExit(0) logging.info('fetching descriptors') - result = fetch(URL_LIST.format(op_project)) + result = fetch(URL_LIST.format(monitoring_project)) descriptors = result.get('metricDescriptors') if not descriptors: raise SystemExit(0) @@ -69,4 +69,4 @@ def main(op_project): if __name__ == '__main__': logging.basicConfig(level=logging.INFO) - main() \ No newline at end of file + main() From 417cbf5fa0860919b440c72f7a891cd9eaa3678a Mon Sep 17 00:00:00 2001 From: Ludo Date: Mon, 12 Dec 2022 07:26:19 +0100 Subject: [PATCH 80/82] dashboard metric rename wip --- .../dashboards/quotas-utilization.json | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index e26d692645..3b6cf44040 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -7,7 +7,7 @@ { "height": 4, "widget": { - "title": "internal_forwarding_rules_l4_utilization", + "title": "Internal L4 forwarding rules utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -24,7 +24,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "1800s", "perSeriesAligner": "ALIGN_MEAN" @@ -47,7 +47,7 @@ { "height": 4, "widget": { - "title": "internal_forwarding_rules_l7_utilization", + "title": "Internal L7 forwarding rules utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -64,7 +64,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/network/forwarding_rules_l4_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" @@ -87,7 +87,7 @@ { "height": 4, "widget": { - "title": "number_of_instances_utilization", + "title": "Instance utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -104,7 +104,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_instances_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/network/instances_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" @@ -127,7 +127,7 @@ { "height": 4, "widget": { - "title": "number_of_vpc_peerings_utilization", + "title": "Peering utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -144,7 +144,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_vpc_peerings_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/network/peerings_total_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" @@ -167,7 +167,7 @@ { "height": 4, "widget": { - "title": "number_of_active_vpc_peerings_utilization", + "title": "Active peering utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -184,7 +184,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_active_vpc_peerings_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/network/peerings_active_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_INTERPOLATE" @@ -207,7 +207,7 @@ { "height": 4, "widget": { - "title": "subnet_IP_ranges_ppg_utilization", + "title": "Subnet IP ranges per peering group utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -224,7 +224,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_subnet_IP_ranges_ppg_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/peering_group/subnet_ranges_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_MEAN" From d25f0b383afb507029db3603e98bb826a819b8ee Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 17 Dec 2022 09:36:46 +0100 Subject: [PATCH 81/82] Update discover-cai-compute.py --- .../network-dashboard/src/plugins/discover-cai-compute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py index 4138a88ad1..86379fb1c7 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py @@ -24,7 +24,6 @@ from . import HTTPRequest, Level, Resource, register_init, register_discovery from .utils import parse_cai_results -# https://content-cloudasset.googleapis.com/v1/organizations/436789450919/assets?contentType=RESOURCE&assetTypes=compute.googleapis.com/Network CAI_URL = ('https://content-cloudasset.googleapis.com/v1' '/{root}/assets' From d85b0bba857adbd55cf05685e6623e775e21067b Mon Sep 17 00:00:00 2001 From: Ludo Date: Sun, 18 Dec 2022 09:55:17 +0100 Subject: [PATCH 82/82] deploy sample dashboard --- .../dashboards/quotas-utilization.json | 231 ++---------------- .../deploy-cloud-function/README.md | 25 +- .../deploy-cloud-function/main.tf | 6 + .../deploy-cloud-function/variables.tf | 6 + 4 files changed, 47 insertions(+), 221 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 3b6cf44040..1c11bdb7af 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -207,7 +207,7 @@ { "height": 4, "widget": { - "title": "Subnet IP ranges per peering group utilization", + "title": "Peering group internal L4 forwarding rules utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -224,47 +224,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/peering_group/subnet_ranges_used_ratio\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "3600s", - "perSeriesAligner": "ALIGN_MEAN" - } - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 0, - "yPos": 16 - }, - { - "height": 4, - "widget": { - "title": "internal_forwarding_rules_l4_ppg_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "3600s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "3600s", - "perSeriesAligner": "ALIGN_NEXT_OLDER" - }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l4_ppg_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/forwarding_rules_l4_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_MEAN" @@ -287,7 +247,7 @@ { "height": 4, "widget": { - "title": "internal_forwarding_rules_l7_ppg_utilization", + "title": "Peering group internal L7 forwarding rules utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -304,7 +264,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/internal_forwarding_rules_l7_ppg_utilization\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/forwarding_rules_l7_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" @@ -327,7 +287,7 @@ { "height": 4, "widget": { - "title": "number_of_instances_ppg_utilization", + "title": "Peering group instance utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -344,7 +304,7 @@ "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/number_of_instances_ppg_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/instances_used_ratio\" resource.type=\"global\"" } } } @@ -363,7 +323,7 @@ { "height": 4, "widget": { - "title": "dynamic_routes_per_network_utilization", + "title": "Peering group dynamic route utilization", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -380,7 +340,7 @@ "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_network_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\"" } } } @@ -399,7 +359,7 @@ { "height": 4, "widget": { - "title": "firewalls_per_project_vpc_usage", + "title": "Project firewall rules used ratio", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -420,7 +380,7 @@ ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_vpc_usage\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\"" } } } @@ -439,47 +399,7 @@ { "height": 4, "widget": { - "title": "firewalls_per_project_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_MAX", - "groupByFields": [ - "metric.label.\"project\"" - ], - "perSeriesAligner": "ALIGN_MAX" - }, - "filter": "metric.type=\"custom.googleapis.com/firewalls_per_project_utilization\" resource.type=\"global\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 6, - "yPos": 32 - }, - { - "height": 4, - "widget": { - "title": "tuples_per_firewall_policy_utilization", + "title": "Firewall policy tuples used ratio", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -496,7 +416,7 @@ "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/firewall_policy_tuples_per_policy_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\"" } } } @@ -515,7 +435,7 @@ { "height": 4, "widget": { - "title": "ip_addresses_per_subnet_utilization", + "title": "IP addressed per subnetwork used ratio", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -532,7 +452,7 @@ "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_subnet_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\"" } } } @@ -551,43 +471,7 @@ { "height": 4, "widget": { - "title": "dynamic_routes_ppg_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"custom.googleapis.com/dynamic_routes_per_peering_group_utilization\" resource.type=\"global\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 6, - "yPos": 20 - }, - { - "height": 4, - "widget": { - "title": "static_routes_per_project_vpc_usage", + "title": "Project static routes used", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -608,7 +492,7 @@ ], "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_vpc_usage\" resource.type=\"global\"", + "filter": "metric.type=\"custom.googleapis.com/netmon/project/routes_static_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_NONE" @@ -632,7 +516,7 @@ { "height": 4, "widget": { - "title": "static_routes_per_ppg_utilization", + "title": "Peering group static routes used", "xyChart": { "chartOptions": { "mode": "COLOR" @@ -649,7 +533,7 @@ "alignmentPeriod": "60s", "perSeriesAligner": "ALIGN_MEAN" }, - "filter": "metric.type=\"custom.googleapis.com/static_routes_per_peering_group_utilization\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\"" } } } @@ -665,85 +549,6 @@ "width": 6, "xPos": 0, "yPos": 28 - }, - { - "height": 4, - "widget": { - "title": "static_routes_per_project_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"custom.googleapis.com/static_routes_per_project_utilization\" resource.type=\"global\"" - } - } - } - ], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 6, - "yPos": 24 - }, - { - "height": 4, - "widget": { - "title": "secondary_ip_address_utilization", - "xyChart": { - "chartOptions": { - "mode": "COLOR" - }, - "dataSets": [ - { - "minAlignmentPeriod": "60s", - "plotType": "LINE", - "targetAxis": "Y1", - "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", - "timeSeriesFilter": { - "aggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_MEAN" - }, - "filter": "metric.type=\"custom.googleapis.com/ip_addresses_per_sr_utilization\" resource.type=\"global\"", - "secondaryAggregation": { - "alignmentPeriod": "60s", - "crossSeriesReducer": "REDUCE_NONE", - "perSeriesAligner": "ALIGN_NONE" - } - } - } - } - ], - "thresholds": [], - "timeshiftDuration": "0s", - "yAxis": { - "label": "y1Axis", - "scale": "LINEAR" - } - } - }, - "width": 6, - "xPos": 0, - "yPos": 36 } ] } diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index a7bbb24431..15288b557d 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -50,22 +50,31 @@ If the function crashes or its behaviour is not as expected, you can turn on deb # copy and paste to the function's "Testing" tab in the console tf output -raw troubleshooting_payload ``` + +## Monitoring dashboard + +A monitoring dashboard can be optionally be deployed int he same project by setting the `dashboard_json_path` variable to the path of a dashboard JSON file. A sample dashboard is in included, and can be deployed with this variable configuration: + +```hcl +dashboard_json_path = "../dashboards/quotas-utilization.json" +``` ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [discovery_config](variables.tf#L38) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empy, every project under the discovery root node will be monitored. | object({…}) | ✓ | | -| [project_id](variables.tf#L84) | Project id where the Cloud Function will be deployed. | string | ✓ | | +| [discovery_config](variables.tf#L44) | Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empy, every project under the discovery root node will be monitored. | object({…}) | ✓ | | +| [project_id](variables.tf#L90) | Project id where the Cloud Function will be deployed. | string | ✓ | | | [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | | [cloud_function_config](variables.tf#L23) | Optional Cloud Function configuration. | object({…}) | | {} | -| [grant_discovery_iam_roles](variables.tf#L56) | Optionally grant required IAM roles to Cloud Function service account. | bool | | false | -| [labels](variables.tf#L63) | Billing labels used for the Cloud Function, and the project if project_create is true. | map(string) | | {} | -| [name](variables.tf#L69) | Name used to create Cloud Function related resources. | string | | "net-dash" | -| [project_create_config](variables.tf#L75) | Optional configuration if project creation is required. | object({…}) | | null | -| [region](variables.tf#L89) | Compute region where the Cloud Function will be deployed. | string | | "europe-west1" | -| [schedule_config](variables.tf#L95) | Schedule timer configuration in crontab format. | string | | "0/30 * * * *" | +| [dashboard_json_path](variables.tf#L38) | Optional monitoring dashboard to deploy. | string | | null | +| [grant_discovery_iam_roles](variables.tf#L62) | Optionally grant required IAM roles to Cloud Function service account. | bool | | false | +| [labels](variables.tf#L69) | Billing labels used for the Cloud Function, and the project if project_create is true. | map(string) | | {} | +| [name](variables.tf#L75) | Name used to create Cloud Function related resources. | string | | "net-dash" | +| [project_create_config](variables.tf#L81) | Optional configuration if project creation is required. | object({…}) | | null | +| [region](variables.tf#L95) | Compute region where the Cloud Function will be deployed. | string | | "europe-west1" | +| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | string | | "0/30 * * * *" | ## Outputs diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf index 887289663d..abbea80e29 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/main.tf @@ -136,3 +136,9 @@ resource "google_project_iam_member" "monitoring" { role = "roles/monitoring.metricWriter" member = module.cloud-function.service_account_iam_email } + +resource "google_monitoring_dashboard" "dashboard" { + count = var.dashboard_json_path == null ? 0 : 1 + project = var.project_id + dashboard_json = file(var.dashboard_json_path) +} diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf index 50e17f5794..ab59f91f52 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf @@ -35,6 +35,12 @@ variable "cloud_function_config" { nullable = false } +variable "dashboard_json_path" { + description = "Optional monitoring dashboard to deploy." + type = string + default = null +} + variable "discovery_config" { description = "Discovery configuration. Discovery root is the organization or a folder. If monitored folders and projects are empy, every project under the discovery root node will be monitored." type = object({