Default cgroup driver to systemd from k8s 1.20

Currently, kOps uses cgroupfs cgroup driver for the kubelet and CRIs. This PR defaults
the cgroup driver to systemd for clusters created with k8s versions >= 1.20.

Using systemd as the cgroup-driver is the recommended way as per
https://kubernetes.io/docs/setup/production-environment/container-runtimes/

docs/cluster_spec.md

@@ -932,6 +932,7 @@ spec:
 ### Configuration
 
 It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options.
+Overriding the configuration of containerd has to be done with care as the default config may change with releases which can lead to incompatibilities.
 
 ```yaml
 spec:
@@ -1178,3 +1179,35 @@ spec:
 ```
 
 which would end up in a drop-in file on all masters and nodes of the cluster.
+
+## cgroupDriver
+
+From k8s 1.20, kOps will default the cgroup driver of the kubelet and all the CRIs to use systemd as the default cgroup driver
+as opposed to cgroup fs.
+
+It is important to ensure that the kubelet and the CRI being used are using the same cgroup driver. Below are examples showing
+how to set the cgroup driver for kubelet and the CRIs currently supported by kOps (docker and containerd). 
+
+Warning: Overriding the configuration of containerd has to be done with care as the default config may change with releases which can lead to incompatibilities.
+
+Setting kubelet to use cgroupfs
+```yaml
+spec:
+  kubelet:
+    cgroupDriver: cgroupfs
+```
+
+Setting docker to use cgroupfs
+```yaml
+spec:
+  docker:
+    execOpt:
+      - native.cgroupdriver=cgroupfs
+```
+
+To set containerd cgroup-driver to cgroupfs, just override the config like below
+```yaml
+spec:
+  containerd:
+    configOverride: ""
+```

nodeup/pkg/model/docker_test.go

@@ -97,6 +97,10 @@ func TestDockerBuilder_BuildFlags(t *testing.T) {
 			kops.DockerConfig{Bridge: fi.String("br0")},
 			"--bridge=br0",
 		},
+		{
+			kops.DockerConfig{ExecOpt: []string{"native.cgroupdriver=systemd"}},
+			"--exec-opt=native.cgroupdriver=systemd",
+		},
 	}
 
 	for _, g := range grid {

nodeup/pkg/model/kubelet.go

@@ -176,6 +176,11 @@ func (b *KubeletBuilder) buildManifestDirectory(kubeletConfig *kops.KubeletConfi
 
 // buildSystemdEnvironmentFile renders the environment file for the kubelet
 func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.KubeletConfigSpec) (*nodetasks.File, error) {
+	// Use systemd as the default cgroup driver from k8s 1.20
+	if b.IsKubernetesGTE("1.20") && kubeletConfig.CgroupDriver == "" {
+		kubeletConfig.CgroupDriver = "systemd"
+	}
+
 	// @step: ensure the masters do not get a bootstrap configuration
 	if b.UseBootstrapTokens() && b.IsMaster {
 		kubeletConfig.BootstrapKubeconfig = ""

pkg/model/components/containerd.go

@@ -1,12 +1,9 @@
 /*
 Copyright 2019 The Kubernetes Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,6 +16,8 @@ package components
 import (
 	"fmt"
 
+	"k8s.io/klog"
+
 	"github.com/blang/semver/v4"
 	"github.com/pelletier/go-toml"
 	"k8s.io/kops/pkg/apis/kops"
@@ -61,7 +60,16 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error {
 			for name, endpoints := range containerd.RegistryMirrors {
 				config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "registry", "mirrors", name, "endpoint"}, endpoints)
 			}
+
+			//default cgroup-driver to systemd from k8s 1.20 onwards
+			if b.IsKubernetesGTE("1.20") {
+				config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc", "runtime_type"}, "io.containerd.runc.v2")
+			}
+
 			containerd.ConfigOverride = fi.String(config.String())
+		} else {
+			klog.Warning("Overriding the configuration of containerd has to be done with care as the default config may" +
+				" change with releases which can lead to incompatibilities.")
 		}
 
 	} else if clusterSpec.ContainerRuntime == "docker" {

pkg/model/components/docker.go

@@ -73,5 +73,10 @@ func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error {
 	// and it is an error to specify the flag twice.
 	docker.Storage = fi.String("overlay2,overlay,aufs")
 
+	// default systemd as cgroup driver in docker from k8s 1.20
+	if b.IsKubernetesGTE("1.20") && len(docker.ExecOpt) == 0 {
+		docker.ExecOpt = append(docker.ExecOpt, "native.cgroupdriver=systemd")
+	}
+
 	return nil
 }

pkg/model/components/kubelet.go

@@ -213,5 +213,10 @@ func (b *KubeletOptionsBuilder) BuildOptions(o interface{}) error {
 		}
 	}
 
+	// default to systemd as cgroup driver for kubelet from k8s 1.20
+	if b.IsKubernetesGTE("1.20") && clusterSpec.Kubelet.CgroupDriver == "" {
+		clusterSpec.Kubelet.CgroupDriver = "systemd"
+	}
+
 	return nil
 }