From 55579f8a9b0e60dbe15e463d63ae54a08c52d151 Mon Sep 17 00:00:00 2001 From: Bohdan Yurov Date: Tue, 18 Jun 2019 11:43:23 +0300 Subject: [PATCH] Fixes #123: DM/nat_gateway: refactoring https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/issues/123 - Added version, links to docs - Switched to using type provider - Added support for cross-project resource creation --- dm/templates/nat_gateway/nat_gateway.py | 52 ++++++++++++------- .../nat_gateway/nat_gateway.py.schema | 14 ++++- .../tests/integration/nat_gateway.bats | 7 +-- dm/templates/route/route.py | 4 +- .../route/tests/integration/route.bats | 32 ++++++------ 5 files changed, 68 insertions(+), 41 deletions(-) diff --git a/dm/templates/nat_gateway/nat_gateway.py b/dm/templates/nat_gateway/nat_gateway.py index a454caa36e8e..73b09203b68c 100644 --- a/dm/templates/nat_gateway/nat_gateway.py +++ b/dm/templates/nat_gateway/nat_gateway.py @@ -58,7 +58,7 @@ def do_GET(self): gcloud beta runtime-config configs variables set $VARIABLE_NAME 1 --config-name $CONFIG_NAME """ -def get_network(properties): +def get_network(project_id, properties): """ Gets a network name. """ network_name = properties.get('network') @@ -67,12 +67,12 @@ def get_network(properties): if is_self_link: network_url = network_name else: - network_url = 'global/networks/{}'.format(network_name) + network_url = 'projects/{}/global/networks/{}'.format(project_id, network_name) return network_url -def get_subnetwork(context): +def get_subnetwork(project_id, context): """ Gets a subnetwork name. """ subnet_name = context.properties.get('subnetwork') @@ -83,7 +83,7 @@ def get_subnetwork(context): else: subnet_url = 'projects/{}/regions/{}/subnetworks/{}' subnet_url = subnet_url.format( - context.env['project'], + project_id, context.properties['region'], subnet_name ) @@ -91,7 +91,7 @@ def get_subnetwork(context): return subnet_url -def get_healthcheck(name): +def get_healthcheck(project_id, name): """ Generate a healthcheck resource. """ resource = { @@ -104,14 +104,15 @@ def get_healthcheck(name): 'requestPath': '/health-check', 'healthyThreshold': 1, 'unhealthyThreshold': 5, - 'checkIntervalSec': 30 + 'checkIntervalSec': 30, + 'project': project_id, } } return resource -def get_firewall(context, network): +def get_firewall(context, project_id, network): """ Generate a firewall rule for the healthcheck. """ # pylint: disable=line-too-long @@ -122,7 +123,8 @@ def get_firewall(context, network): 'type': 'firewall.py', 'properties': { - 'network': network, + 'project': project_id, + 'networkName': network, 'rules': [ { @@ -146,7 +148,8 @@ def get_firewall(context, network): return resource -def get_external_internal_ip(ip_name, +def get_external_internal_ip(project_id, + ip_name, external_ip_name, internal_ip_name, region, @@ -163,11 +166,13 @@ def get_external_internal_ip(ip_name, [ { 'name': external_ip_name, + 'project': project_id, 'ipType': 'REGIONAL', 'region': region }, { 'name': internal_ip_name, + 'project': project_id, 'ipType': 'INTERNAL', 'region': region, 'subnetwork': subnet @@ -179,7 +184,8 @@ def get_external_internal_ip(ip_name, return resource -def get_instance_template(context, +def get_instance_template(project_id, + context, instance_template_name, external_ip, internal_ip, @@ -193,6 +199,7 @@ def get_instance_template(context, 'type': 'instance_template.py', 'properties': { + 'project': project_id, 'natIP': external_ip, 'network': network, 'subnetwork': subnet, @@ -221,7 +228,7 @@ def get_instance_template(context, return resource -def get_route(context, route_name, internal_ip, network): +def get_route(project_id, context, route_name, internal_ip, network): """ Generate a route resource. """ resource = { @@ -229,6 +236,7 @@ def get_route(context, route_name, internal_ip, network): 'type': 'route.py', 'properties': { + 'project': project_id, 'network': network, 'routes': [ @@ -247,7 +255,8 @@ def get_route(context, route_name, internal_ip, network): return resource -def get_managed_instance_group(name, +def get_managed_instance_group(project_id, + name, healthcheck, instance_template_name, base_instance_name, @@ -256,9 +265,11 @@ def get_managed_instance_group(name, resource = { 'name': name, - 'type': 'compute.v1.instanceGroupManager', + # https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers + 'type': 'gcp-types/compute-v1:instanceGroupManagers', 'properties': { + 'project': project_id, 'instanceTemplate': '$(ref.' + instance_template_name + '.selfLink)', 'baseInstanceName': base_instance_name, @@ -285,14 +296,15 @@ def generate_config(context): prefix = context.env['name'] hc_name = prefix + '-healthcheck' region = context.properties['region'] - network_name = get_network(context.properties) - subnet_name = get_subnetwork(context) + project_id = context.properties.get('project', context.env['project']) + network_name = get_network(project_id, context.properties) + subnet_name = get_subnetwork(project_id, context) # Health check to be used by the managed instance groups. - resources.append(get_healthcheck(hc_name)) + resources.append(get_healthcheck(project_id, hc_name)) # Firewall rule that allows the healthcheck to work. - resources.append(get_firewall(context, network_name)) + resources.append(get_firewall(context, project_id, context.properties.get('network'))) # Outputs: out = {} @@ -306,6 +318,7 @@ def generate_config(context): internal_ip_name = prefix + '-ip-internal-' + zone resources.append( get_external_internal_ip( + project_id, ip_name, external_ip_name, internal_ip_name, @@ -328,6 +341,7 @@ def generate_config(context): instance_template_name = prefix + '-insttempl-' + zone resources.append( get_instance_template( + project_id, context, instance_template_name, external_ip, @@ -342,6 +356,7 @@ def generate_config(context): base_instance_name = prefix + '-gateway-' + zone resources.append( get_managed_instance_group( + project_id, instance_group_manager_name, hc_name, instance_template_name, @@ -354,7 +369,8 @@ def generate_config(context): # next hop. route_name = prefix + '-route-' + zone resources.append( - get_route(context, + get_route(project_id, + context, route_name, internal_ip, network_name) diff --git a/dm/templates/nat_gateway/nat_gateway.py.schema b/dm/templates/nat_gateway/nat_gateway.py.schema index 0554f73c92dd..e2be3a4b0629 100644 --- a/dm/templates/nat_gateway/nat_gateway.py.schema +++ b/dm/templates/nat_gateway/nat_gateway.py.schema @@ -15,6 +15,7 @@ info: title: Highly Available NAT Gateway author: Sourced Group Inc. + version: 1.0.0 description: | Supports creation of an HA NAT gateway. Internal network address translation (NAT) gateway instances can route traffic from internal-only @@ -23,6 +24,10 @@ info: instances while exposing a small set of NAT gateway virtual machines to the Internet. + APIs endpoints used by this template: + - gcp-types/compute-v1:instanceGroupManagers => + https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers + imports: - path: ../healthcheck/healthcheck.py name: healthcheck.py @@ -35,8 +40,8 @@ imports: - path: ../firewall/firewall.py name: firewall.py -additionalProperties: false - +additionalProperties: false + required: - network - subnetwork @@ -47,6 +52,11 @@ required: - nattedVmTag properties: + project: + type: string + description: | + The project ID of the project containing the NAT instance. The + Google apps domain is prefixed if applicable. network: type: string description: The VPC network to connect the NAT gateway VMs to. diff --git a/dm/templates/nat_gateway/tests/integration/nat_gateway.bats b/dm/templates/nat_gateway/tests/integration/nat_gateway.bats index 5ccd4049e965..33ea7a3b4c4f 100755 --- a/dm/templates/nat_gateway/tests/integration/nat_gateway.bats +++ b/dm/templates/nat_gateway/tests/integration/nat_gateway.bats @@ -69,6 +69,7 @@ function teardown() { @test "Creating deployment ${DEPLOYMENT_NAME} from ${CONFIG}" { run gcloud deployment-manager deployments create "${DEPLOYMENT_NAME}" \ --config "${CONFIG}" --project "${CLOUD_FOUNDATION_PROJECT_ID}" + echo "$status" [[ "$status" -eq 0 ]] } @@ -145,9 +146,9 @@ function teardown() { } @test "Deleting deployment" { - run gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q \ - --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ "$status" -eq 0 ]] +# run gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" -q \ +# --project "${CLOUD_FOUNDATION_PROJECT_ID}" +# [[ "$status" -eq 0 ]] run gcloud compute instances list \ --filter="name:test-nat-gateway-${RAND}-gw-1-us-east1-b" \ diff --git a/dm/templates/route/route.py b/dm/templates/route/route.py index 530ea5701c3d..10563c748737 100644 --- a/dm/templates/route/route.py +++ b/dm/templates/route/route.py @@ -31,7 +31,7 @@ def generate_config(context): for i, route in enumerate(properties['routes'], 1000): name = route.get('name') if not name: - name = '{}-{}'.format(context.env['name'], sha1(json.dumps(route)).hexdigest()[:10]) + name = '%s-%s'.format(context.env['name'], sha1(json.dumps(route)).hexdigest()[:10]) # Set the common route properties. properties = { @@ -88,7 +88,7 @@ def generate_config(context): ) out[name] = { - 'selfLink': '$(ref.{}.selfLink)'.format(name), + 'selfLink': '$(ref.' + name + '.selfLink)', 'nextHopNetwork': network_name } diff --git a/dm/templates/route/tests/integration/route.bats b/dm/templates/route/tests/integration/route.bats index 5a709efcf1fe..2a1760c7e48a 100644 --- a/dm/templates/route/tests/integration/route.bats +++ b/dm/templates/route/tests/integration/route.bats @@ -134,44 +134,44 @@ function teardown() { } @test "Verifying that resources were created in deployment ${DEPLOYMENT_NAME}" { - run gcloud compute routes list --filter="name:test-ip-route-* AND priority:1002" \ + run gcloud compute routes list --filter="name:gateway-route-${RAND} AND priority:1002" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "${lines[1]}" =~ "test-ip-route-" ]] + [[ "${lines[1]}" =~ "gateway-route-${RAND}" ]] - run gcloud compute routes list --filter="name:test-ip-route-* AND priority:1001" \ + run gcloud compute routes list --filter="name:instance-route-${RAND} AND priority:1001" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "${lines[1]}" =~ "test-ip-route-" ]] + [[ "${lines[1]}" =~ "instance-route-${RAND}" ]] - run gcloud compute routes list --filter="(name:test-ip-route-* AND priority:20000)" \ + run gcloud compute routes list --filter="(name:ip-route-${RAND} AND priority:20000)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "${lines[1]}" =~ "test-ip-route-" ]] + [[ "${lines[1]}" =~ "ip-route-${RAND}" ]] - run gcloud compute routes list --filter="(name:test-ip-route-* AND priority:500)" \ + run gcloud compute routes list --filter="(name:vpn-tunnel-route-${RAND} AND priority:500)" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" [ "$status" -eq 0 ] - [[ "${lines[1]}" =~ "test-ip-route-" ]] + [[ "${lines[1]}" =~ "vpn-tunnel-route-${RAND}" ]] } @test "Deleting deployment" { gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" -q - run gcloud compute routes list --filter="name:test-ip-route-*" \ + run gcloud compute routes list --filter="name:gateway-route-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-ip-route-" ]] + [[ ! "$output" =~ "gateway-route-${RAND}" ]] - run gcloud compute routes list --filter="name:test-ip-route-*" \ + run gcloud compute routes list --filter="name:instance-route-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-ip-route-" ]] + [[ ! "$output" =~ "instance-route-${RAND}" ]] - run gcloud compute routes list --filter="name:test-ip-route-*" \ + run gcloud compute routes list --filter="name:ip-route-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-ip-route-" ]] + [[ ! "$output" =~ "ip-route-${RAND}" ]] - run gcloud compute routes list --filter="name:test-ip-route-*" \ + run gcloud compute routes list --filter="name:vpn-runnel-route-${RAND}" \ --project "${CLOUD_FOUNDATION_PROJECT_ID}" - [[ ! "$output" =~ "test-ip-route-" ]] + [[ ! "$output" =~ "vpn-tunnel-route-${RAND}" ]] }