mTLS + local client side balancing for gRPC

CMakeLists.txt

@@ -20,28 +20,30 @@ add_custom_target(cpy ALL
   WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
 )
 
+#
+# key generation 
+#
+add_custom_target(key ALL
+  openssl genrsa -out rootkey.pem 4096
+  WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+)
+
 #
 # images
 #
 add_custom_target(images ALL
-  sudo mkosi --force
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/images
+sudo ./build.sh
+WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/images
 )
 
 #
 # delegatio-agent
 #
 add_custom_target(delegatio-agent ALL
-  go build -o ${CMAKE_BINARY_DIR}/delegatio-agent
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/agent/vm
-  BYPRODUCTS delegatio-agent
+go build -o ${CMAKE_BINARY_DIR}/delegatio-agent
+WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/agent/vm
+BYPRODUCTS delegatio-agent
 )
 
 add_dependencies(cpy delegatio-agent)
-
-
-add_custom_target(ssh ALL
-  go build -o ${CMAKE_BINARY_DIR}/ssh
-  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/ssh
-  BYPRODUCTS ssh
-)
+add_dependencies(images cpy)

agent/container/main.go

@@ -8,10 +8,9 @@ import (
 	"flag"
 	"log"
 	"net"
+	"strconv"
 
-	"cloud.google.com/go/compute/metadata"
 	"github.com/benschlueter/delegatio/internal/config"
-	"github.com/benschlueter/delegatio/internal/config/definitions"
 	grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
 	"go.uber.org/zap"
 )
@@ -29,7 +28,6 @@ func main() {
 	cfg := zap.NewDevelopmentConfig()
 
 	logLevelUser := flag.Bool("debug", false, "enables gRPC debug output")
-	containerMode := flag.Bool("container", false, "signals that the agent is running in a container")
 	flag.Parse()
 	cfg.Level.SetLevel(zap.DebugLevel)
 
@@ -45,36 +43,8 @@ func main() {
 	zapLoggerCore := zapLogger.Named("core")
 
 	bindIP = config.DefaultIP
-	bindPort = config.PublicAPIport
+	bindPort = strconv.Itoa(config.AgentPort)
 	dialer := &net.Dialer{}
 
-	var ipAddr string
-	if metadata.OnGCE() {
-		ipAddr, err = metadata.InstanceAttributeValue("loadbalancer")
-		if err != nil {
-			zapLoggerCore.Fatal("failed to get loadbalancer ip from metadata | not running in cloud", zap.Error(err))
-		}
-
-		localIP, err := metadata.InternalIP()
-		if err != nil {
-			zapLoggerCore.Fatal("failed to get local ip from metadata", zap.Error(err))
-		}
-		zapLoggerCore.Info("local ip", zap.String("ip", localIP))
-
-		attr, err := metadata.ProjectAttributes()
-		if err != nil {
-			zapLoggerCore.Fatal("failed to get project attributes from metadata", zap.Error(err))
-		}
-		zapLoggerCore.Info("project attributes", zap.Any("attributes", attr))
-
-		iattr, err := metadata.InstanceAttributes()
-		if err != nil {
-			zapLoggerCore.Fatal("failed to get instance attributes from metadata", zap.Error(err))
-		}
-		zapLoggerCore.Info("instance attributes", zap.Any("attributes", iattr))
-	} else {
-		ipAddr = definitions.NetworkXMLConfig.IPs[0].Address
-	}
-
-	run(dialer, bindIP, bindPort, zapLoggerCore, containerMode, ipAddr)
+	run(dialer, bindIP, bindPort, zapLoggerCore)
 }

agent/container/run.go

@@ -7,14 +7,12 @@
 package main
 
 import (
-	"context"
 	"net"
 	"sync"
 
+	"github.com/benschlueter/delegatio/agent/container/core"
 	"github.com/benschlueter/delegatio/agent/manageapi"
 	"github.com/benschlueter/delegatio/agent/manageapi/manageproto"
-	"github.com/benschlueter/delegatio/agent/vm/core"
-	"github.com/benschlueter/delegatio/agent/vm/core/state"
 	"github.com/benschlueter/delegatio/internal/config"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap"
@@ -31,17 +29,13 @@ var version = "0.0.0"
  * via the loadbalancerIPAddr to give us the join token. At the same time
  * we are waiting for the init-request from a user *only* if we are a control plane.
  */
-func run(dialer manageapi.Dialer, bindIP, bindPort string, zapLoggerCore *zap.Logger, containerMode *bool, loadbalancerIPAddr string) {
+func run(dialer manageapi.Dialer, bindIP, bindPort string, zapLoggerCore *zap.Logger) {
 	defer func() { _ = zapLoggerCore.Sync() }()
 	zapLoggerCore.Info("starting delegatio agent", zap.String("version", version), zap.String("commit", config.Commit))
 
-	if *containerMode {
-		zapLoggerCore.Info("running in container mode")
-	} else {
-		zapLoggerCore.Info("running in qemu mode")
-	}
+	zapLoggerCore.Info("running in qemu mode")
 
-	core, err := core.NewCore(zapLoggerCore, loadbalancerIPAddr)
+	core, err := core.NewCore(zapLoggerCore)
 	if err != nil {
 		zapLoggerCore.Fatal("failed to create core", zap.Error(err))
 	}
@@ -67,8 +61,6 @@ func run(dialer manageapi.Dialer, bindIP, bindPort string, zapLoggerCore *zap.Lo
 		zapLoggergRPC.Fatal("failed to create listener", zap.Error(err))
 	}
 	zapLoggergRPC.Info("server listener created", zap.String("address", lis.Addr().String()))
-	core.State.Advance(state.AcceptingInit)
-	go core.TryJoinCluster(context.Background())
 
 	var wg sync.WaitGroup
 	defer wg.Wait()

agent/vm/core/core.go

@@ -18,11 +18,8 @@ import (
 	"time"
 
 	"github.com/benschlueter/delegatio/agent/vm/core/state"
-	"github.com/benschlueter/delegatio/agent/vm/vmapi/vmproto"
-	"github.com/benschlueter/delegatio/internal/config"
+	"github.com/benschlueter/delegatio/agent/vm/vmapi"
 	"go.uber.org/zap"
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials/insecure"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
@@ -44,15 +41,17 @@ type Core struct {
 	mux                  sync.Mutex
 	client               clientset.Interface
 	masterLoadbalancerIP string
+	api                  vmapi.VMAPI
 }
 
 // NewCore creates and initializes a new Core object.
-func NewCore(zapLogger *zap.Logger, loadbalancerIP string) (*Core, error) {
+func NewCore(zapLogger *zap.Logger, loadbalancerIP string, api vmapi.VMAPI) (*Core, error) {
 	c := &Core{
 		zaplogger:            zapLogger,
 		masterLoadbalancerIP: loadbalancerIP,
 		mux:                  sync.Mutex{},
 		client:               nil,
+		api:                  api,
 	}
 	return c, nil
 }
@@ -138,15 +137,7 @@ func (c *Core) JoinCluster(ctx context.Context) error {
 	if err := c.State.Require(state.AcceptingInit); err != nil {
 		return err
 	}
-	c.zaplogger.Info("dial master")
-	conn, err := grpc.DialContext(ctx, net.JoinHostPort(c.masterLoadbalancerIP, config.PublicAPIport), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock())
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-	client := vmproto.NewAPIClient(conn)
-	c.zaplogger.Info("grpc call to master")
-	joinData, err := client.GetJoinDataKube(ctx, &vmproto.GetJoinDataKubeRequest{})
+	joinData, err := c.api.GetJoinData(ctx)
 	if err != nil {
 		return err
 	}
@@ -202,20 +193,20 @@ func (c *Core) executeKubeadm(ctx context.Context, endpoint, token, caCert strin
 }
 
 // TryJoinCluster tries to join the cluster every 5 seconds until it succeeds.
-func (c *Core) TryJoinCluster(ctx context.Context) {
+func (c *Core) TryJoinCluster(ctx context.Context) error {
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-ticker.C:
 			if c.State.Get() >= state.JoiningCluster {
-				return
+				return nil
 			}
 			if err := c.JoinCluster(ctx); err != nil {
 				c.zaplogger.Info("Failed to join cluster, retrying in 5 seconds", zap.Error(err))
 			}
 		case <-ctx.Done():
-			return
+			return nil
 		}
 	}
 }

agent/vm/main.go

@@ -8,6 +8,7 @@ import (
 	"flag"
 	"log"
 	"net"
+	"strconv"
 
 	"cloud.google.com/go/compute/metadata"
 	"github.com/benschlueter/delegatio/internal/config"
@@ -25,7 +26,7 @@ import (
  * CLI rpc calls.
  */
 func main() {
-	var bindIP, bindPort string
+	var bindIP, agentPort, vmPort string
 	cfg := zap.NewDevelopmentConfig()
 
 	logLevelUser := flag.Bool("debug", false, "enables gRPC debug output")
@@ -45,7 +46,8 @@ func main() {
 	zapLoggerCore := zapLogger.Named("core")
 
 	bindIP = config.DefaultIP
-	bindPort = config.PublicAPIport
+	agentPort = strconv.Itoa(config.AgentPort)
+	vmPort = strconv.Itoa(config.VMAgentPort)
 	dialer := &net.Dialer{}
 
 	var ipAddr string
@@ -76,5 +78,5 @@ func main() {
 		ipAddr = definitions.NetworkXMLConfig.IPs[0].Address
 	}
 
-	run(dialer, bindIP, bindPort, zapLoggerCore, containerMode, ipAddr)
+	run(dialer, bindIP, agentPort, vmPort, zapLoggerCore, containerMode, ipAddr)
 }