From abbf60ae27246ad5c8f5de7c4304633d82818bd0 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Sat, 9 May 2020 11:37:31 +0300 Subject: [PATCH] Add support for Kubenet with containerd --- nodeup/pkg/model/containerd.go | 42 ++++++++++++++++++- .../tests/containerdbuilder/simple/tasks.yaml | 24 +++++++++++ pkg/model/components/containerd.go | 18 +++++++- .../cloudformation.json.extracted.yaml | 14 ++++++- 4 files changed, 94 insertions(+), 4 deletions(-) diff --git a/nodeup/pkg/model/containerd.go b/nodeup/pkg/model/containerd.go index 7bbeb3480f63f..1128ee6648420 100644 --- a/nodeup/pkg/model/containerd.go +++ b/nodeup/pkg/model/containerd.go @@ -335,6 +335,13 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { return err } + // Using containerd with Kubenet requires special configuration. This is a temporary backwards-compatible solution + // and will be deprecated when Kubenet is deprecated: + // https://github.com/containerd/cri/blob/master/docs/config.md#cni-config-template + if b.Cluster.Spec.ContainerRuntime == "containerd" && b.Cluster.Spec.Networking != nil && b.Cluster.Spec.Networking.Kubenet != nil { + b.buildKubenetConflistTemplate(c) + } + return nil } @@ -389,7 +396,6 @@ func (b *ContainerdBuilder) buildContainerOSConfigurationDropIn(c *fi.ModelBuild "EnvironmentFile=/etc/environment", "TasksMax=infinity", } - contents := strings.Join(lines, "\n") c.AddTask(&nodetasks.File{ @@ -441,6 +447,40 @@ func (b *ContainerdBuilder) buildSysconfig(c *fi.ModelBuilderContext) error { return nil } +// buildKubenetConflistTemplate is responsible for creating a special template for setups using Kubenet +func (b *ContainerdBuilder) buildKubenetConflistTemplate(c *fi.ModelBuilderContext) { + lines := []string{ + "{", + " \"cniVersion\": \"0.3.1\",", + " \"name\": \"kubenet\",", + " \"plugins\": [", + " {", + " \"type\": \"bridge\",", + " \"bridge\": \"cbr0\",", + " \"mtu\": 1460,", + " \"addIf\": \"eth0\",", + " \"isGateway\": true,", + " \"ipMasq\": true,", + " \"promiscMode\": true,", + " \"ipam\": {", + " \"type\": \"host-local\",", + " \"subnet\": \"{{.PodCIDR}}\",", + " \"routes\": [{ \"dst\": \"0.0.0.0/0\" }]", + " }", + " }", + " ]", + "}", + } + contents := strings.Join(lines, "\n") + klog.V(8).Infof("Built kubenet CNI config file\n%s", contents) + + c.AddTask(&nodetasks.File{ + Path: "/etc/containerd/kubenet.conflist.template", + Contents: fi.NewStringResource(contents), + Type: nodetasks.FileType_File, + }) +} + // skipInstall determines if kops should skip the installation and configuration of containerd func (b *ContainerdBuilder) skipInstall() bool { d := b.Cluster.Spec.Containerd diff --git a/nodeup/pkg/model/tests/containerdbuilder/simple/tasks.yaml b/nodeup/pkg/model/tests/containerdbuilder/simple/tasks.yaml index 11717a8360000..08e231758bd16 100644 --- a/nodeup/pkg/model/tests/containerdbuilder/simple/tasks.yaml +++ b/nodeup/pkg/model/tests/containerdbuilder/simple/tasks.yaml @@ -2,6 +2,30 @@ contents: "" path: /etc/containerd/config-kops.toml type: file --- +contents: |- + { + "cniVersion": "0.3.1", + "name": "kubenet", + "plugins": [ + { + "type": "bridge", + "bridge": "cbr0", + "mtu": 1460, + "addIf": "eth0", + "isGateway": true, + "ipMasq": true, + "promiscMode": true, + "ipam": { + "type": "host-local", + "subnet": "{{.PodCIDR}}", + "routes": [{ "dst": "0.0.0.0/0" }] + } + } + ] + } +path: /etc/containerd/kubenet.conflist.template +type: file +--- contents: CONTAINERD_OPTS= path: /etc/sysconfig/containerd type: file diff --git a/pkg/model/components/containerd.go b/pkg/model/components/containerd.go index 2123ab52a131d..5bc430116edd3 100644 --- a/pkg/model/components/containerd.go +++ b/pkg/model/components/containerd.go @@ -18,6 +18,7 @@ package components import ( "fmt" + "strings" "k8s.io/klog" "k8s.io/kops/pkg/apis/kops" @@ -62,7 +63,22 @@ func (b *ContainerdOptionsBuilder) BuildOptions(o interface{}) error { // Apply defaults for containerd running in container runtime mode containerd.LogLevel = fi.String("info") - containerd.ConfigOverride = fi.String("") + if clusterSpec.Networking != nil && clusterSpec.Networking.Kubenet != nil { + // Using containerd with Kubenet requires special configuration. This is a temporary backwards-compatible solution + // and will be deprecated when Kubenet is deprecated: + // https://github.com/containerd/cri/blob/master/docs/config.md#cni-config-template + lines := []string{ + "version = 2", + "[plugins]", + " [plugins.\"io.containerd.grpc.v1.cri\"]", + " [plugins.\"io.containerd.grpc.v1.cri\".cni]", + " conf_template = \"/etc/containerd/kubenet.conflist.template\"", + } + contents := strings.Join(lines, "\n") + containerd.ConfigOverride = fi.String(contents) + } else { + containerd.ConfigOverride = fi.String("") + } } else if clusterSpec.ContainerRuntime == "docker" { if fi.StringValue(containerd.Version) == "" { diff --git a/tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml index 378e316501b31..791d7830aa329 100644 --- a/tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml @@ -134,7 +134,12 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amasterscontainerdexampl cloudConfig: null containerRuntime: containerd containerd: - configOverride: "" + configOverride: |- + version = 2 + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".cni] + conf_template = "/etc/containerd/kubenet.conflist.template" logLevel: info version: 1.2.10 docker: @@ -421,7 +426,12 @@ Resources.AWSAutoScalingLaunchConfigurationnodescontainerdexamplecom.Properties. cloudConfig: null containerRuntime: containerd containerd: - configOverride: "" + configOverride: |- + version = 2 + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".cni] + conf_template = "/etc/containerd/kubenet.conflist.template" logLevel: info version: 1.2.10 docker: