Support Private DNS HostedZones via FeatureFlag

pkg/featureflag/featureflag.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package featureflag
+
+import (
+	"os"
+	"strings"
+	"sync"
+)
+
+var PreviewPrivateDNS = New("PreviewPrivateDNS")
+
+var flags = make(map[string]*FeatureFlag)
+var flagsMutex sync.Mutex
+
+var initFlags sync.Once
+
+type FeatureFlag struct {
+	Key     string
+	Enabled bool
+}
+
+func readFlags() {
+	f := os.Getenv("KOPS_FEATURE_FLAGS")
+	f = strings.TrimSpace(f)
+	for _, s := range strings.Split(f, ",") {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		ff := New(s)
+		ff.Enabled = true
+	}
+
+}
+
+func New(key string) *FeatureFlag {
+	initFlags.Do(readFlags)
+
+	flagsMutex.Lock()
+	defer flagsMutex.Unlock()
+
+	f := flags[key]
+	if f == nil {
+		f = &FeatureFlag{
+			Key: key,
+		}
+		flags[key] = f
+	}
+
+	return f
+}

pkg/model/context.go

@@ -18,7 +18,9 @@ package model
 
 import (
 	"fmt"
+	"github.com/golang/glog"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/featureflag"
 	"strings"
 )
 
@@ -151,3 +153,11 @@ func (m *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma
 func (m *KopsModelContext) UseLoadBalancerForAPI() bool {
 	return m.Cluster.Spec.Topology.Masters == kops.TopologyPrivate
 }
+
+func (m *KopsModelContext) UsePrivateDNS() bool {
+	if featureflag.PreviewPrivateDNS.Enabled {
+		glog.Infof("PreviewPrivateDNS enabled; using private DNS")
+		return true
+	}
+	return false
+}

pkg/model/dns.go

@@ -29,23 +29,41 @@ type DNSModelBuilder struct {
 var _ fi.ModelBuilder = &DNSModelBuilder{}
 
 func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error {
-	if b.UseLoadBalancerForAPI() {
+	if b.UsePrivateDNS() {
+		// This is only exposed as a feature flag currently
+		// TODO: We may still need a public zone to publish an ELB
+
+		// Configuration for a DNS zone, attached to our VPC
+		dnsZone := &awstasks.DNSZone{
+			Name:       s(b.Cluster.Spec.DNSZone),
+			Private:    fi.Bool(true),
+			PrivateVPC: b.LinkToVPC(),
+		}
+		c.AddTask(dnsZone)
+	} else if b.UseLoadBalancerForAPI() {
 		// This will point our DNS to the load balancer, and put the pieces
 		// together for kubectl to be work
 
 		// Configuration for a DNS name for the master
 		dnsZone := &awstasks.DNSZone{
-			Name: s(b.Cluster.Spec.DNSZone),
+			Name:    s(b.Cluster.Spec.DNSZone),
+			Private: fi.Bool(false),
 		}
 		c.AddTask(dnsZone)
+	}
+
+	if b.UseLoadBalancerForAPI() {
+		// This will point our DNS to the load balancer, and put the pieces
+		// together for kubectl to be work
 
 		dnsName := &awstasks.DNSName{
 			Name:               s(b.Cluster.Spec.MasterPublicName),
-			Zone:               dnsZone,
+			Zone:               &awstasks.DNSZone{Name: s(b.Cluster.Spec.DNSZone)},
 			ResourceType:       s("A"),
 			TargetLoadBalancer: b.LinkToELB("api"),
 		}
 		c.AddTask(dnsName)
 	}
+
 	return nil
 }

upup/pkg/fi/cloudup/apply_cluster.go

@@ -577,6 +577,19 @@ func findHash(url string) (*hashing.Hash, error) {
 }
 
 func validateDNS(cluster *api.Cluster, cloud fi.Cloud) error {
+	kopsModelContext := &model.KopsModelContext{
+		//Region: cloud.Region(),
+		Cluster: cluster,
+		//InstanceGroups []*kops.InstanceGroup
+
+		//SSHPublicKeys [][]byte
+	}
+
+	if kopsModelContext.UsePrivateDNS() {
+		glog.Infof("Private DNS: skipping DNS validation")
+		return nil
+	}
+
 	dns, err := cloud.DNS()
 	if err != nil {
 		return fmt.Errorf("error building DNS provider: %v", err)

upup/pkg/fi/cloudup/awstasks/dnszone.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
 	"math/rand"
+	"reflect"
 	"strconv"
 	"strings"
 )
@@ -34,6 +35,9 @@ import (
 type DNSZone struct {
 	Name *string
 	ID   *string
+
+	Private    *bool
+	PrivateVPC *VPC
 }
 
 var _ fi.CompareWithID = &DNSZone{}
@@ -56,7 +60,22 @@ func (e *DNSZone) Find(c *fi.Context) (*DNSZone, error) {
 
 	actual := &DNSZone{}
 	actual.Name = e.Name
-	actual.ID = z.Id
+	actual.ID = z.HostedZone.Id
+	actual.Private = z.HostedZone.Config.PrivateZone
+
+	// If the zone is private, but we don't want it to be, that will be an error
+	// e.PrivateVPC won't be set, so we can't find the "right" VPC (without cheating)
+	if e.PrivateVPC != nil {
+		for _, vpc := range z.VPCs {
+			if cloud.Region() != aws.StringValue(vpc.VPCRegion) {
+				continue
+			}
+
+			if aws.StringValue(e.PrivateVPC.ID) == aws.StringValue(vpc.VPCId) {
+				actual.PrivateVPC = e.PrivateVPC
+			}
+		}
+	}
 
 	if e.ID == nil {
 		e.ID = actual.ID
@@ -65,7 +84,7 @@ func (e *DNSZone) Find(c *fi.Context) (*DNSZone, error) {
 	return actual, nil
 }
 
-func (e *DNSZone) findExisting(cloud awsup.AWSCloud) (*route53.HostedZone, error) {
+func (e *DNSZone) findExisting(cloud awsup.AWSCloud) (*route53.GetHostedZoneOutput, error) {
 	findID := ""
 	if e.ID != nil {
 		findID = *e.ID
@@ -89,7 +108,7 @@ func (e *DNSZone) findExisting(cloud awsup.AWSCloud) (*route53.HostedZone, error
 				return nil, fmt.Errorf("error fetching DNS HostedZone %q: %v", findID, err)
 			}
 		} else {
-			return response.HostedZone, nil
+			return response, nil
 		}
 	}
 
@@ -115,14 +134,23 @@ func (e *DNSZone) findExisting(cloud awsup.AWSCloud) (*route53.HostedZone, error
 			zones = append(zones, zone)
 		}
 	}
+
 	if len(zones) == 0 {
 		return nil, nil
-	}
-	if len(zones) != 1 {
+	} else if len(zones) != 1 {
 		return nil, fmt.Errorf("found multiple hosted zones matched name %q", findName)
-	}
+	} else {
+		request := &route53.GetHostedZoneInput{
+			Id: zones[0].Id,
+		}
+
+		response, err := cloud.Route53().GetHostedZone(request)
+		if err != nil {
+			return nil, fmt.Errorf("error fetching DNS HostedZone by id %q: %v", *request.Id, err)
+		}
 
-	return zones[0], nil
+		return response, nil
+	}
 }
 
 func (e *DNSZone) Run(c *fi.Context) error {
@@ -137,20 +165,52 @@ func (s *DNSZone) CheckChanges(a, e, changes *DNSZone) error {
 }
 
 func (_ *DNSZone) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSZone) error {
+	name := aws.StringValue(e.Name)
 	if a == nil {
 		request := &route53.CreateHostedZoneInput{}
 		request.Name = e.Name
 		nonce := rand.Int63()
 		request.CallerReference = aws.String(strconv.FormatInt(nonce, 10))
 
-		glog.V(2).Infof("Creating Route53 HostedZone with Name %q", e.Name)
+		if e.PrivateVPC != nil {
+			request.VPC = &route53.VPC{
+				VPCId:     e.PrivateVPC.ID,
+				VPCRegion: aws.String(t.Cloud.Region()),
+			}
+		}
+
+		glog.V(2).Infof("Creating Route53 HostedZone with Name %q", name)
 
 		response, err := t.Cloud.Route53().CreateHostedZone(request)
 		if err != nil {
-			return fmt.Errorf("error creating DNS HostedZone: %v", err)
+			return fmt.Errorf("error creating DNS HostedZone %q: %v", name, err)
 		}
 
 		e.ID = response.HostedZone.Id
+	} else {
+		if changes.PrivateVPC != nil {
+			request := &route53.AssociateVPCWithHostedZoneInput{
+				HostedZoneId: a.ID,
+				VPC: &route53.VPC{
+					VPCId:     e.PrivateVPC.ID,
+					VPCRegion: aws.String(t.Cloud.Region()),
+				},
+			}
+
+			changes.PrivateVPC = nil
+
+			glog.V(2).Infof("Updating DNSZone %q", name)
+
+			_, err := t.Cloud.Route53().AssociateVPCWithHostedZone(request)
+			if err != nil {
+				return fmt.Errorf("error associating VPC with hosted zone %q: %v", name, err)
+			}
+		}
+
+		empty := &DNSZone{}
+		if !reflect.DeepEqual(empty, changes) {
+			glog.Warningf("cannot apply changes to DNSZone %q: %v", name, changes)
+		}
 	}
 
 	// We don't tag the zone - we expect it to be shared
@@ -159,6 +219,7 @@ func (_ *DNSZone) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSZone) error
 
 type terraformRoute53Zone struct {
 	Name      *string              `json:"name"`
+	VPCID     *terraform.Literal   `json:"vpc_id,omitempty"`
 	Tags      map[string]string    `json:"tags,omitempty"`
 	Lifecycle *terraform.Lifecycle `json:"lifecycle,omitempty"`
 }
@@ -176,9 +237,9 @@ func (_ *DNSZone) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *D
 	}
 
 	if z != nil {
-		glog.Infof("Existing zone %q found; will configure TF to reuse", aws.StringValue(z.Name))
+		glog.Infof("Existing zone %q found; will configure TF to reuse", aws.StringValue(z.HostedZone.Name))
 
-		e.ID = z.Id
+		e.ID = z.HostedZone.Id
 	}
 
 	if z == nil {
@@ -200,6 +261,11 @@ func (_ *DNSZone) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *D
 		//
 		//return t.RenderResource("aws_route53_zone", *e.Name, tf)
 	} else {
+		// Same problem here also...
+		if e.PrivateVPC != nil {
+			return fmt.Errorf("Route53 private hosted zones are not supported for terraform")
+		}
+
 		return nil
 	}
 }