Merge pull request #10540 from hakman/flatcar-containerd-config

Add containerd config file to Flatcar based instances
kubernetes · Jan 7, 2021 · 59c6aaa · 59c6aaa
2 parents 4e08ed9 + 422cfad
commit 59c6aaa
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 40 deletions.
diff --git a/nodeup/pkg/model/containerd.go b/nodeup/pkg/model/containerd.go
@@ -51,16 +51,14 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error {
 	switch b.Distribution {
 	case distributions.DistributionFlatcar:
 		klog.Infof("Detected Flatcar; won't install containerd")
-		if err := b.buildContainerOSConfigurationDropIn(c); err != nil {
-			return err
+		if b.Cluster.Spec.ContainerRuntime == "containerd" {
+			b.buildSystemdServiceOverrideFlatcar(c)
+			b.buildConfigFile(c)
 		}
 		return nil
-
 	case distributions.DistributionContainerOS:
 		klog.Infof("Detected ContainerOS; won't install containerd")
-		if err := b.buildContainerOSConfigurationDropIn(c); err != nil {
-			return err
-		}
+		b.buildSystemdServiceOverrideContainerOS(c)
 		return nil
 	}
 
@@ -75,19 +73,7 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 
 	// Add config file
-	{
-		containerdConfigOverride := ""
-		if b.Cluster.Spec.Containerd != nil {
-			containerdConfigOverride = fi.StringValue(b.Cluster.Spec.Containerd.ConfigOverride)
-		}
-
-		t := &nodetasks.File{
-			Path:     "/etc/containerd/config-kops.toml",
-			Contents: fi.NewStringResource(containerdConfigOverride),
-			Type:     nodetasks.FileType_File,
-		}
-		c.AddTask(t)
-	}
+	b.buildConfigFile(c)
 
 	// Add binaries from assets
 	if b.Cluster.Spec.ContainerRuntime == "containerd" {
@@ -129,7 +115,7 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error {
 	}
 	c.AddTask(b.buildSystemdService(sv))
 
-	if err := b.buildSysconfig(c); err != nil {
+	if err := b.buildSysconfigFile(c); err != nil {
 		return err
 	}
 
@@ -191,41 +177,58 @@ func (b *ContainerdBuilder) buildSystemdService(sv semver.Version) *nodetasks.Se
 	return service
 }
 
-// buildContainerOSConfigurationDropIn is responsible for configuring the containerd daemon options
-func (b *ContainerdBuilder) buildContainerOSConfigurationDropIn(c *fi.ModelBuilderContext) error {
+// buildSystemdServiceOverrideContainerOS is responsible for overriding the containerd service for ContainerOS
+func (b *ContainerdBuilder) buildSystemdServiceOverrideContainerOS(c *fi.ModelBuilderContext) {
 	lines := []string{
 		"[Service]",
-		"EnvironmentFile=/etc/sysconfig/containerd",
 		"EnvironmentFile=/etc/environment",
 		"TasksMax=infinity",
 	}
 	contents := strings.Join(lines, "\n")
 
 	c.AddTask(&nodetasks.File{
-		AfterFiles: []string{"/etc/sysconfig/containerd"},
-		Path:       "/etc/systemd/system/containerd.service.d/10-kops.conf",
-		Contents:   fi.NewStringResource(contents),
-		Type:       nodetasks.FileType_File,
+		Path:     "/etc/systemd/system/containerd.service.d/10-kops.conf",
+		Contents: fi.NewStringResource(contents),
+		Type:     nodetasks.FileType_File,
 		OnChangeExecute: [][]string{
 			{"systemctl", "daemon-reload"},
 			{"systemctl", "restart", "containerd.service"},
 			// We need to restart kops-configuration service since nodeup needs to load images
-			// into containerd with the new config. Restart is on the background because
-			// kops-configuration is of type 'one-shot' so the restart command will wait for
-			// nodeup to finish executing
+			// into containerd with the new config. We restart in the background because
+			// kops-configuration is of type "one-shot", so the restart command will wait for
+			// nodeup to finish executing.
 			{"systemctl", "restart", "kops-configuration.service", "&"},
 		},
 	})
+}
 
-	if err := b.buildSysconfig(c); err != nil {
-		return err
+// buildSystemdServiceOverrideFlatcar is responsible for overriding the containerd service for Flatcar
+func (b *ContainerdBuilder) buildSystemdServiceOverrideFlatcar(c *fi.ModelBuilderContext) {
+	lines := []string{
+		"[Service]",
+		"Environment=CONTAINERD_CONFIG=/etc/containerd/config-kops.toml",
+		"EnvironmentFile=/etc/environment",
 	}
+	contents := strings.Join(lines, "\n")
 
-	return nil
+	c.AddTask(&nodetasks.File{
+		Path:     "/etc/systemd/system/containerd.service.d/10-kops.conf",
+		Contents: fi.NewStringResource(contents),
+		Type:     nodetasks.FileType_File,
+		OnChangeExecute: [][]string{
+			{"systemctl", "daemon-reload"},
+			{"systemctl", "restart", "containerd.service"},
+			// We need to restart kops-configuration service since nodeup needs to load images
+			// into containerd with the new config. We restart in the background because
+			// kops-configuration is of type "one-shot", so the restart command will wait for
+			// nodeup to finish executing.
+			{"systemctl", "restart", "kops-configuration.service", "&"},
+		},
+	})
 }
 
-// buildSysconfig is responsible for extracting the containerd configuration and writing the sysconfig file
-func (b *ContainerdBuilder) buildSysconfig(c *fi.ModelBuilderContext) error {
+// buildSysconfigFile is responsible for creating the containerd sysconfig file
+func (b *ContainerdBuilder) buildSysconfigFile(c *fi.ModelBuilderContext) error {
 	var containerd kops.ContainerdConfig
 	if b.Cluster.Spec.Containerd != nil {
 		containerd = *b.Cluster.Spec.Containerd
@@ -250,6 +253,20 @@ func (b *ContainerdBuilder) buildSysconfig(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
+// buildConfigFile is responsible for creating the containerd configuration file
+func (b *ContainerdBuilder) buildConfigFile(c *fi.ModelBuilderContext) {
+	containerdConfigOverride := ""
+	if b.Cluster.Spec.Containerd != nil {
+		containerdConfigOverride = fi.StringValue(b.Cluster.Spec.Containerd.ConfigOverride)
+	}
+
+	c.AddTask(&nodetasks.File{
+		Path:     "/etc/containerd/config-kops.toml",
+		Contents: fi.NewStringResource(containerdConfigOverride),
+		Type:     nodetasks.FileType_File,
+	})
+}
+
 // skipInstall determines if kops should skip the installation and configuration of containerd
 func (b *ContainerdBuilder) skipInstall() bool {
 	d := b.Cluster.Spec.Containerd

diff --git a/nodeup/pkg/model/containerd_test.go b/nodeup/pkg/model/containerd_test.go
@@ -29,15 +29,19 @@ import (
 )
 
 func TestContainerdBuilder_Docker_19_03_13(t *testing.T) {
-	runContainerdBuilderTest(t, "from_docker_19.03.11")
+	runContainerdBuilderTest(t, "from_docker_19.03.11", distributions.DistributionUbuntu2004)
 }
 
 func TestContainerdBuilder_Docker_19_03_14(t *testing.T) {
-	runContainerdBuilderTest(t, "from_docker_19.03.14")
+	runContainerdBuilderTest(t, "from_docker_19.03.14", distributions.DistributionUbuntu2004)
 }
 
 func TestContainerdBuilder_Simple(t *testing.T) {
-	runContainerdBuilderTest(t, "simple")
+	runContainerdBuilderTest(t, "simple", distributions.DistributionUbuntu2004)
+}
+
+func TestContainerdBuilder_Flatcar(t *testing.T) {
+	runContainerdBuilderTest(t, "flatcar", distributions.DistributionFlatcar)
 }
 
 func TestContainerdBuilder_SkipInstall(t *testing.T) {
@@ -123,7 +127,7 @@ func TestContainerdBuilder_BuildFlags(t *testing.T) {
 	}
 }
 
-func runContainerdBuilderTest(t *testing.T, key string) {
+func runContainerdBuilderTest(t *testing.T, key string, distro distributions.Distribution) {
 	basedir := path.Join("tests/containerdbuilder/", key)
 
 	nodeUpModelContext, err := BuildNodeupModelContext(basedir)
@@ -132,7 +136,7 @@ func runContainerdBuilderTest(t *testing.T, key string) {
 		return
 	}
 
-	nodeUpModelContext.Distribution = distributions.DistributionUbuntu1604
+	nodeUpModelContext.Distribution = distro
 
 	nodeUpModelContext.Assets = fi.NewAssetStore("")
 	nodeUpModelContext.Assets.AddForTest("containerd", "usr/local/bin/containerd", "testing containerd content")

diff --git a/nodeup/pkg/model/tests/containerdbuilder/flatcar/cluster.yaml b/nodeup/pkg/model/tests/containerdbuilder/flatcar/cluster.yaml
@@ -0,0 +1,39 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+    - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  containerRuntime: containerd
+  containerd:
+    version: 1.4.3
+  etcdClusters:
+    - etcdMembers:
+        - instanceGroup: master-us-test-1a
+          name: master-us-test-1a
+      name: main
+    - etcdMembers:
+        - instanceGroup: master-us-test-1a
+          name: master-us-test-1a
+      name: events
+  kubernetesVersion: v1.19.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+    - cidr: 172.20.32.0/19
+      name: us-test-1a
+      type: Public
+      zone: us-test-1a
diff --git a/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml b/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml
@@ -0,0 +1,20 @@
+contents: ""
+path: /etc/containerd/config-kops.toml
+type: file
+---
+contents: |-
+  [Service]
+  Environment=CONTAINERD_CONFIG=/etc/containerd/config-kops.toml
+  EnvironmentFile=/etc/environment
+onChangeExecute:
+- - systemctl
+  - daemon-reload
+- - systemctl
+  - restart
+  - containerd.service
+- - systemctl
+  - restart
+  - kops-configuration.service
+  - '&'
+path: /etc/systemd/system/containerd.service.d/10-kops.conf
+type: file