Add a bastion host and place instances + kube apiserver behind it

aws/container-linux/kubernetes/bastion.tf

@@ -0,0 +1,166 @@
+resource "aws_autoscaling_group" "bastion" {
+  name = "${var.cluster_name}-bastion ${aws_launch_configuration.bastion.name}"
+
+  # count
+  desired_capacity          = "${var.bastion_count}"
+  min_size                  = "${var.bastion_count}"
+  max_size                  = "${var.bastion_count}"
+  default_cooldown          = 30
+  health_check_grace_period = 30
+
+  # network
+  vpc_zone_identifier = ["${aws_subnet.private.*.id}"]
+
+  # template
+  launch_configuration = "${aws_launch_configuration.bastion.name}"
+
+  # target groups to which instances should be added
+  target_group_arns = [
+    "${aws_lb_target_group.bastion.id}"
+  ]
+
+  lifecycle {
+    # override the default destroy and replace update behavior
+    create_before_destroy = true
+    ignore_changes        = ["image_id"]
+  }
+
+  tags = [{
+    key                 = "Name"
+    value               = "${var.cluster_name}-bastion"
+    propagate_at_launch = true
+  }]
+}
+
+resource "aws_launch_configuration" "bastion" {
+  image_id      = "${data.aws_ami.coreos.image_id}"
+  instance_type = "${var.bastion_type}"
+
+  user_data = "${data.ct_config.bastion_ign.rendered}"
+
+  # network
+  security_groups = [
+    "${aws_security_group.bastion_external.id}"
+  ]
+
+  lifecycle {
+    // Override the default destroy and replace update behavior
+    create_before_destroy = true
+  }
+}
+
+data "template_file" "bastion_config" {
+  template = "${file("${path.module}/cl/bastion.yaml.tmpl")}"
+
+  vars = {
+    ssh_authorized_key = "${var.ssh_authorized_key}"
+  }
+}
+
+data "ct_config" "bastion_ign" {
+  content = "${data.template_file.bastion_config.rendered}"
+  pretty_print = false
+}
+
+resource "aws_security_group" "bastion_external" {
+  name_prefix = "${var.cluster_name}-bastion-external-"
+  description = "Allows access to the bastion from the internet"
+
+  vpc_id = "${aws_vpc.network.id}"
+
+  tags {
+    Name = "${var.cluster_name}-bastion-external"
+  }
+
+  ingress {
+    protocol = "tcp"
+    from_port = 22
+    to_port = 22
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    protocol = -1
+    from_port = 0
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_security_group" "bastion_internal" {
+  name_prefix = "${var.cluster_name}-bastion-internal-"
+  description = "Allows access to a host from the bastion"
+
+  vpc_id = "${aws_vpc.network.id}"
+
+  tags {
+    Name = "${var.cluster_name}-bastion-internal"
+  }
+
+  ingress {
+    protocol = "tcp"
+    from_port = 22
+    to_port = 22
+    security_groups = ["${aws_security_group.bastion_external.id}"]
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_lb" "bastion" {
+  name               = "${var.cluster_name}-bastion"
+  load_balancer_type = "network"
+
+  subnets = ["${aws_subnet.public.*.id}"]
+}
+
+
+resource "aws_lb_listener" "bastion" {
+  load_balancer_arn = "${aws_lb.bastion.arn}"
+  protocol          = "TCP"
+  port              = "22"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${aws_lb_target_group.bastion.arn}"
+  }
+}
+
+resource "aws_lb_target_group" "bastion" {
+  name        = "${var.cluster_name}-bastion"
+  vpc_id      = "${aws_vpc.network.id}"
+  target_type = "instance"
+
+  protocol = "TCP"
+  port     = 22
+
+  health_check {
+    protocol = "TCP"
+    port     = 22
+
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+
+    interval = 10
+  }
+}
+
+resource "aws_route53_record" "bastion" {
+  zone_id = "${var.dns_zone_id}"
+
+  name = "${format("bastion.%s.%s.", var.cluster_name, var.dns_zone)}"
+  type = "A"
+
+  # AWS recommends their special "alias" records for ELBs
+  alias {
+    name                   = "${aws_lb.bastion.dns_name}"
+    zone_id                = "${aws_lb.bastion.zone_id}"
+    evaluate_target_health = false
+  }
+}

aws/container-linux/kubernetes/cl/bastion.yaml.tmpl

@@ -0,0 +1,6 @@
+---
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - "${ssh_authorized_key}"

aws/container-linux/kubernetes/controllers.tf

@@ -33,9 +33,11 @@ resource "aws_instance" "controllers" {
   }
 
   # network
-  associate_public_ip_address = true
-  subnet_id                   = "${element(aws_subnet.public.*.id, count.index)}"
-  vpc_security_group_ids      = ["${aws_security_group.controller.id}"]
+  subnet_id = "${element(aws_subnet.private.*.id, count.index)}"
+  vpc_security_group_ids = [
+    "${aws_security_group.controller.id}",
+    "${aws_security_group.bastion_internal.id}"
+  ]
 
   lifecycle {
     ignore_changes = ["ami"]

aws/container-linux/kubernetes/elb.tf

@@ -7,37 +7,61 @@ resource "aws_route53_record" "apiserver" {
 
   # AWS recommends their special "alias" records for ELBs
   alias {
-    name                   = "${aws_elb.apiserver.dns_name}"
-    zone_id                = "${aws_elb.apiserver.zone_id}"
+    name                   = "${aws_lb.apiserver.dns_name}"
+    zone_id                = "${aws_lb.apiserver.zone_id}"
     evaluate_target_health = true
   }
 }
 
-# Controller Network Load Balancer
-resource "aws_elb" "apiserver" {
-  name            = "${var.cluster_name}-apiserver"
-  subnets         = ["${aws_subnet.public.*.id}"]
-  security_groups = ["${aws_security_group.controller.id}"]
-
-  listener {
-    lb_port           = 443
-    lb_protocol       = "tcp"
-    instance_port     = 443
-    instance_protocol = "tcp"
+# Network Load Balancer for apiservers
+resource "aws_lb" "apiserver" {
+  name               = "${var.cluster_name}-apiserver"
+  load_balancer_type = "network"
+  internal           = true
+
+  subnets = ["${aws_subnet.private.*.id}"]
+}
+
+# Forward HTTP traffic to controllers
+resource "aws_lb_listener" "apiserver-https" {
+  load_balancer_arn = "${aws_lb.apiserver.arn}"
+  protocol          = "TCP"
+  port              = "443"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = "${aws_lb_target_group.controllers.arn}"
   }
+}
 
-  instances = ["${aws_instance.controllers.*.id}"]
+# Target group of controllers
+resource "aws_lb_target_group" "controllers" {
+  name        = "${var.cluster_name}-controllers"
+  vpc_id      = "${aws_vpc.network.id}"
+  target_type = "ip"
+
+  protocol = "TCP"
+  port     = 443
 
   # Kubelet HTTP health check
   health_check {
-    target              = "SSL:443"
-    healthy_threshold   = 2
-    unhealthy_threshold = 4
-    timeout             = 5
-    interval            = 6
+    protocol = "TCP"
+    port     = 443
+
+    # NLBs required to use same healthy and unhealthy thresholds
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+
+    # Interval between health checks required to be 10 or 30
+    interval = 10
   }
+}
+
+# Attach controller instances to apiserver NLB
+resource "aws_lb_target_group_attachment" "controllers" {
+  count = "${var.controller_count}"
 
-  idle_timeout                = 3600
-  connection_draining         = true
-  connection_draining_timeout = 300
+  target_group_arn = "${aws_lb_target_group.controllers.arn}"
+  target_id        = "${element(aws_instance.controllers.*.private_ip, count.index)}"
+  port             = 443
 }

aws/container-linux/kubernetes/network.tf

@@ -17,7 +17,7 @@ resource "aws_internet_gateway" "gateway" {
   tags = "${map("Name", "${var.cluster_name}")}"
 }
 
-resource "aws_route_table" "default" {
+resource "aws_route_table" "public" {
   vpc_id = "${aws_vpc.network.id}"
 
   route {
@@ -30,7 +30,7 @@ resource "aws_route_table" "default" {
     gateway_id      = "${aws_internet_gateway.gateway.id}"
   }
 
-  tags = "${map("Name", "${var.cluster_name}")}"
+  tags = "${map("Name", "${var.cluster_name}-public")}"
 }
 
 # Subnets (one per availability zone)
@@ -52,6 +52,60 @@ resource "aws_subnet" "public" {
 resource "aws_route_table_association" "public" {
   count = "${length(data.aws_availability_zones.all.names)}"
 
-  route_table_id = "${aws_route_table.default.id}"
+  route_table_id = "${aws_route_table.public.id}"
   subnet_id      = "${element(aws_subnet.public.*.id, count.index)}"
 }
+
+resource "aws_subnet" "private" {
+  count = "${length(data.aws_availability_zones.all.names)}"
+
+  vpc_id            = "${aws_vpc.network.id}"
+  availability_zone = "${data.aws_availability_zones.all.names[count.index]}"
+
+  cidr_block                      = "${cidrsubnet(var.host_cidr, 4, count.index + 8)}"
+  ipv6_cidr_block                 = "${cidrsubnet(aws_vpc.network.ipv6_cidr_block, 8, count.index + 8)}"
+  assign_ipv6_address_on_creation = true
+
+  tags = "${map("Name", "${var.cluster_name}-private-${count.index}")}"
+}
+
+resource "aws_route_table" "private" {
+  vpc_id = "${aws_vpc.network.id}"
+
+  route {
+    cidr_block     = "0.0.0.0/0"
+    nat_gateway_id = "${aws_nat_gateway.nat.id}"
+  }
+
+  route {
+    ipv6_cidr_block        =   "::/0"
+    egress_only_gateway_id = "${aws_egress_only_internet_gateway.egress_igw.id}"
+  }
+
+  tags = "${map("Name", "${var.cluster_name}-private")}"
+}
+
+resource "aws_route_table_association" "private" {
+  count = "${length(data.aws_availability_zones.all.names)}"
+
+  route_table_id = "${aws_route_table.private.id}"
+  subnet_id      = "${element(aws_subnet.private.*.id, count.index)}"
+}
+
+
+resource "aws_eip" "nat" {
+  vpc = true
+}
+
+resource "aws_nat_gateway" "nat" {
+  depends_on = [
+    "aws_internet_gateway.gateway",
+  ]
+
+  allocation_id = "${aws_eip.nat.id}"
+  subnet_id = "${element(aws_subnet.public.*.id, 0)}"
+}
+
+resource "aws_egress_only_internet_gateway" "egress_igw" {
+  vpc_id = "${aws_vpc.network.id}"
+}

aws/container-linux/kubernetes/ssh.tf

@@ -4,8 +4,13 @@ resource "null_resource" "copy-secrets" {
 
   connection {
     type    = "ssh"
-    host    = "${element(aws_instance.controllers.*.public_ip, count.index)}"
+
+    host    = "${element(aws_instance.controllers.*.private_ip, count.index)}"
     user    = "core"
+
+    bastion_host = "${aws_route53_record.bastion.fqdn}"
+    bastion_user = "core"
+
     timeout = "15m"
   }
 
@@ -73,8 +78,13 @@ resource "null_resource" "bootkube-start" {
 
   connection {
     type    = "ssh"
-    host    = "${aws_instance.controllers.0.public_ip}"
+
+    host    = "${aws_instance.controllers.0.private_ip}"
     user    = "core"
+
+    bastion_host = "${aws_route53_record.bastion.fqdn}"
+    bastion_user = "core"
+
     timeout = "15m"
   }