add internal loadbalancer for azure

pkg/model/azuremodel/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/apis/kops:go_default_library",
+        "//pkg/dns:go_default_library",
         "//pkg/model:go_default_library",
         "//pkg/model/defaults:go_default_library",
         "//pkg/model/iam:go_default_library",
@@ -24,12 +25,14 @@ go_library(
         "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
     srcs = [
+        "api_loadbalancer_test.go",
         "context_test.go",
         "network_test.go",
         "resourcegroup_test.go",

pkg/model/azuremodel/api_loadbalancer.go

@@ -17,9 +17,14 @@ limitations under the License.
 package azuremodel
 
 import (
-	"errors"
+	"fmt"
 
+	"github.com/Azure/go-autorest/autorest/to"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/azuretasks"
 )
 
 // APILoadBalancerModelBuilder builds a LoadBalancer for accessing the API
@@ -38,5 +43,81 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		return nil
 	}
 
-	return errors.New("using loadbalancer for API server is not yet implemented in Azure")
+	lbSpec := b.Cluster.Spec.API.LoadBalancer
+	if lbSpec == nil {
+		// Skipping API LB creation; not requested in Spec
+		return nil
+	}
+
+	switch lbSpec.Type {
+	case kops.LoadBalancerTypeInternal:
+		// OK
+	case kops.LoadBalancerTypePublic:
+		// TODO: Implement creating public ip and attach to public loadbalancer
+		return fmt.Errorf("only internal loadbalancer for API server is implemented in Azure")
+	default:
+		return fmt.Errorf("unhandled LoadBalancer type %q", lbSpec.Type)
+	}
+
+	// Create LoadBalancer for API ELB
+	lb := &azuretasks.LoadBalancer{
+		Name:          fi.String(b.NameForLoadBalancer()),
+		Lifecycle:     b.Lifecycle,
+		ResourceGroup: b.LinkToResourceGroup(),
+		Tags:          map[string]*string{},
+	}
+
+	switch lbSpec.Type {
+	case kops.LoadBalancerTypeInternal:
+		lb.External = to.BoolPtr(false)
+		subnet, err := b.subnetForLoadBalancer()
+		if err != nil {
+			return err
+		}
+		lb.Subnet = b.LinkToAzureSubnet(subnet)
+	case kops.LoadBalancerTypePublic:
+		lb.External = to.BoolPtr(true)
+	default:
+		return fmt.Errorf("unknown load balancer Type: %q", lbSpec.Type)
+	}
+
+	c.AddTask(lb)
+
+	if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() {
+		lb.ForAPIServer = true
+	}
+
+	return nil
+}
+
+// subnetForLoadBalancer returns the subnet the loadbalancer will use.
+func (c *AzureModelContext) subnetForLoadBalancer() (*kops.ClusterSubnetSpec, error) {
+	var subnets []*kops.ClusterSubnetSpec
+	// Get all private subnets in cluster spec
+	for i := range c.Cluster.Spec.Subnets {
+		subnet := &c.Cluster.Spec.Subnets[i]
+
+		if c.Cluster.Spec.API.LoadBalancer.Type == kops.LoadBalancerTypeInternal && subnet.Type != kops.SubnetTypePrivate {
+			continue
+		}
+
+		subnets = append(subnets, subnet)
+	}
+
+	// Get all master instance group subnets
+	migSubnets := sets.NewString()
+	for _, ig := range c.MasterInstanceGroups() {
+		for _, subnet := range ig.Spec.Subnets {
+			migSubnets.Insert(subnet)
+		}
+	}
+
+	// Return first master subnet found
+	for _, subnet := range subnets {
+		if migSubnets.Has(subnet.Name) {
+			return subnet, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no suitable subnets found")
 }

pkg/model/azuremodel/api_loadbalancer_test.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuremodel
+
+import (
+	"reflect"
+	"testing"
+
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+)
+
+func TestAPILoadBalancerModelBuilder_Build(t *testing.T) {
+	b := APILoadBalancerModelBuilder{
+		AzureModelContext: newTestAzureModelContext(),
+	}
+	c := &fi.ModelBuilderContext{
+		Tasks: make(map[string]fi.Task),
+	}
+	err := b.Build(c)
+	if err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
+}
+
+func TestSubnetForLoadbalancer(t *testing.T) {
+	b := APILoadBalancerModelBuilder{
+		AzureModelContext: newTestAzureModelContext(),
+	}
+	b.Cluster.Spec.Subnets = []kops.ClusterSubnetSpec{
+		{
+			Name: "master",
+			Type: kops.SubnetTypePrivate,
+		},
+		{
+			Name: "node",
+			Type: kops.SubnetTypePrivate,
+		},
+		{
+			Name: "utility",
+			Type: kops.SubnetTypeUtility,
+		},
+	}
+	b.InstanceGroups[0].Spec.Role = kops.InstanceGroupRoleMaster
+	b.InstanceGroups[0].Spec.Subnets = []string{
+		"master",
+	}
+
+	actual, err := b.subnetForLoadBalancer()
+	if err != nil {
+		t.Error(err)
+	}
+	expected := &kops.ClusterSubnetSpec{
+		Name: "master",
+		Type: kops.SubnetTypePrivate,
+	}
+	if !reflect.DeepEqual(actual, expected) {
+		t.Errorf("expected subnet %+v, but got %+v", expected, actual)
+	}
+
+}

pkg/model/azuremodel/context.go

@@ -66,6 +66,16 @@ func (c *AzureModelContext) NameForRouteTable() string {
 	return c.Cluster.Spec.CloudConfig.Azure.RouteTableName
 }
 
+// LinkToLoadBalancer returns the Load Balancer object for the cluster.
+func (c *AzureModelContext) LinkToLoadBalancer() *azuretasks.LoadBalancer {
+	return &azuretasks.LoadBalancer{Name: fi.String(c.NameForLoadBalancer())}
+}
+
+// NameForLoadBalancer returns the name of the Load Balancer object for the cluster.
+func (c *AzureModelContext) NameForLoadBalancer() string {
+	return "api-" + c.ClusterName()
+}
+
 // CloudTagsForInstanceGroup computes the tags to apply to instances in the specified InstanceGroup
 // Mostly copied from pkg/model/context.go, but "/" in tag keys are replaced with "_" as Azure
 // doesn't allow "/" in tag keys.

pkg/model/azuremodel/testing.go

@@ -44,6 +44,11 @@ func newTestCluster() *kops.Cluster {
 			Name: "testcluster.test.com",
 		},
 		Spec: kops.ClusterSpec{
+			API: &kops.AccessSpec{
+				LoadBalancer: &kops.LoadBalancerAccessSpec{
+					Type: kops.LoadBalancerTypeInternal,
+				},
+			},
 			NetworkID:   "test-virtual-network",
 			NetworkCIDR: "10.0.0.0/8",
 			Subnets: []kops.ClusterSubnetSpec{

pkg/model/azuremodel/vmscaleset.go

@@ -126,6 +126,19 @@ func (b *VMScaleSetModelBuilder) buildVMScaleSetTask(
 		return nil, fmt.Errorf("unexpected subnet type: for InstanceGroup %q; type was %s", ig.Name, subnet.Type)
 	}
 
+	switch ig.Spec.Role {
+	case kops.InstanceGroupRoleMaster:
+		t.RequireLoadBalancer = fi.Bool(false)
+		if b.Cluster.Spec.API.LoadBalancer != nil {
+			t.RequireLoadBalancer = fi.Bool(true)
+			t.LoadBalancerName = to.StringPtr(b.NameForLoadBalancer())
+		}
+	case kops.InstanceGroupRoleNode:
+		t.RequireLoadBalancer = fi.Bool(false)
+	default:
+		return nil, fmt.Errorf("unexpected role type: for InstanceGroup %q; type was %s", ig.Name, ig.Spec.Role)
+	}
+
 	t.Tags = b.CloudTagsForInstanceGroup(ig)
 
 	return t, nil

pkg/resources/azure/azure.go

@@ -39,6 +39,7 @@ const (
 	typeVMScaleSet     = "VMScaleSet"
 	typeDisk           = "Disk"
 	typeRoleAssignment = "RoleAssignment"
+	typeLoadBalancer   = "LoadBalancer"
 )
 
 // ListResourcesAzure lists all resources for the cluster by quering Azure.
@@ -87,6 +88,7 @@ func (g *resourceGetter) listAll() ([]*resources.Resource, error) {
 		g.listRouteTables,
 		g.listVMScaleSetsAndRoleAssignments,
 		g.listDisks,
+		g.listLoadBalancers,
 	}
 
 	var resources []*resources.Resource
@@ -395,6 +397,38 @@ func (g *resourceGetter) deleteRoleAssignment(_ fi.Cloud, r *resources.Resource)
 	return g.cloud.RoleAssignment().Delete(context.TODO(), *ra.Scope, *ra.Name)
 }
 
+func (g *resourceGetter) listLoadBalancers(ctx context.Context) ([]*resources.Resource, error) {
+	loadBalancers, err := g.cloud.LoadBalancer().List(ctx, g.resourceGroupName())
+	if err != nil {
+		return nil, err
+	}
+
+	var rs []*resources.Resource
+	for i := range loadBalancers {
+		rt := &loadBalancers[i]
+		if !g.isOwnedByCluster(rt.Tags) {
+			continue
+		}
+		rs = append(rs, g.toLoadBalancerResource(rt))
+	}
+	return rs, nil
+}
+
+func (g *resourceGetter) toLoadBalancerResource(loadBalancer *network.LoadBalancer) *resources.Resource {
+	return &resources.Resource{
+		Obj:     loadBalancer,
+		Type:    typeLoadBalancer,
+		ID:      *loadBalancer.Name,
+		Name:    *loadBalancer.Name,
+		Deleter: g.deleteLoadBalancer,
+		Blocks:  []string{toKey(typeResourceGroup, g.resourceGroupName())},
+	}
+}
+
+func (g *resourceGetter) deleteLoadBalancer(_ fi.Cloud, r *resources.Resource) error {
+	return g.cloud.LoadBalancer().Delete(context.TODO(), g.resourceGroupName(), r.Name)
+}
+
 // isOwnedByCluster returns true if the resource is owned by the cluster.
 func (g *resourceGetter) isOwnedByCluster(tags map[string]*string) bool {
 	for k, v := range tags {

pkg/resources/azure/azure_test.go

@@ -46,6 +46,7 @@ func TestListResourcesAzure(t *testing.T) {
 		raName         = "ra"
 		irrelevantName = "irrelevant"
 		principalID    = "pid"
+		lbName         = "lb"
 	)
 	clusterTags := map[string]*string{
 		azure.TagClusterName: to.StringPtr(clusterName),
@@ -163,6 +164,15 @@ func TestListResourcesAzure(t *testing.T) {
 		Name: to.StringPtr(irrelevantName),
 	}
 
+	lbs := cloud.LoadBalancersClient.LBs
+	lbs[lbName] = network.LoadBalancer{
+		Name: to.StringPtr(lbName),
+		Tags: clusterTags,
+	}
+	lbs[irrelevantName] = network.LoadBalancer{
+		Name: to.StringPtr(irrelevantName),
+	}
+
 	// Call listResourcesAzure.
 	g := resourceGetter{
 		cloud: cloud,
@@ -253,6 +263,11 @@ func TestListResourcesAzure(t *testing.T) {
 				toKey(typeVMScaleSet, vmssName),
 			},
 		},
+		toKey(typeLoadBalancer, lbName): {
+			rtype:  typeLoadBalancer,
+			name:   lbName,
+			blocks: []string{toKey(typeResourceGroup, rgName)},
+		},
 	}
 	if !reflect.DeepEqual(a, e) {
 		t.Errorf("expected %+v, but got %+v", e, a)

upup/pkg/fi/cloudup/azure/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
         "azure_cloud.go",
         "azure_utils.go",
         "disk.go",
+        "loadbalancer.go",
         "networkinterface.go",
         "resourcegroup.go",
         "roleassignment.go",