Skip to content

Commit

Permalink
fix: set config_path in Containerd config
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkoshkin
dkoshkin committed Feb 15, 2024
1 parent 17fa13f commit c8ba1cf
Show file tree
Hide file tree
Showing 6 changed files with 220 additions and 9 deletions.
41 changes: 34 additions & 7 deletions pkg/handlers/generic/mutation/mirrors/inject.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func (h *globalMirrorPatchHandler) Mutate(
if err != nil {
return err
}
files, generateErr := generateFiles(
files, commands, generateErr := generateFilesAndCommands(
mirrorConfig,
globalMirror)
if generateErr != nil {
Expand All @@ -115,6 +115,15 @@ func (h *globalMirrorPatchHandler) Mutate(
files...,
)

log.WithValues(
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding PreKubeadmCommands to control plane kubeadm config spec")
obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands,
commands...,
)

return nil
}); err != nil {
return err
Expand All @@ -132,7 +141,7 @@ func (h *globalMirrorPatchHandler) Mutate(
if err != nil {
return err
}
files, generateErr := generateFiles(
files, commands, generateErr := generateFilesAndCommands(
mirrorConfig,
globalMirror)
if generateErr != nil {
Expand All @@ -144,6 +153,13 @@ func (h *globalMirrorPatchHandler) Mutate(
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding global registry mirror files to worker node kubeadm config template")
obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, files...)

log.WithValues(
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
).Info("adding PreKubeadmCommands to worker node kubeadm config template")
obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, commands...)

return nil
}); err != nil {
return err
Expand All @@ -152,18 +168,29 @@ func (h *globalMirrorPatchHandler) Mutate(
return nil
}

func generateFiles(
func generateFilesAndCommands(
mirrorConfig *mirrorConfig,
globalMirror v1alpha1.GlobalImageRegistryMirror,
) ([]bootstrapv1.File, error) {
// Generate default registry mirror file
) ([]bootstrapv1.File, []string, error) {
// generate default registry mirror file
files, err := generateGlobalRegistryMirrorFile(mirrorConfig)
if err != nil {
return nil, err
return nil, nil, err
}
// generate CA certificate file for registry mirror
mirrorCAFile := generateMirrorCACertFile(mirrorConfig, globalMirror)
files = append(files, mirrorCAFile...)
// generate Containerd registry config drop-in file
registryConfigDropIn := generateContainerdRegistryConfigDropInFile()
files = append(files, registryConfigDropIn...)
// generate Containerd apply patches script and command
var commands []string
applyPatchesFile, applyPatchesCommand, err := generateContainerdApplyPatchesScript()
if err != nil {
return nil, nil, err
}
files = append(files, applyPatchesFile...)
commands = append(commands, applyPatchesCommand)

return files, err
return files, commands, err
}
59 changes: 57 additions & 2 deletions pkg/handlers/generic/mutation/mirrors/mirror.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
_ "embed"
"fmt"
"path"
"text/template"

corev1 "k8s.io/api/core/v1"
Expand All @@ -21,6 +22,11 @@ const (
mirrorCACertPathOnRemote = "/etc/certs/mirror.pem"
defaultRegistryMirrorConfigPathOnRemote = "/etc/containerd/certs.d/_default/hosts.toml"
secretKeyForMirrorCACert = "ca.crt"

tomlMergeImage = "ghcr.io/mesosphere/toml-merge:v0.2.0"
containerdPatchesDirOnRemote = "/etc/containerd/cre.d"
containerdApplyPatchesScriptOnRemote = "/etc/containerd/apply-patches.sh"
containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote
)

var (
Expand All @@ -30,6 +36,16 @@ var (
defaultRegistryMirrorPatchTemplate = template.Must(
template.New("").Parse(string(defaultRegistryMirrorPatch)),
)

//go:embed templates/containerd-registry-config-drop-in.toml
containerdRegistryConfigDropIn []byte
containerdRegistryConfigDropInFileOnRemote = path.Join(
containerdPatchesDirOnRemote,
"registry-config.toml",
)

//go:embed templates/containerd-apply-patches.sh.gotmpl
containerdApplyConfigPatchesScript []byte
)

type mirrorConfig struct {
Expand Down Expand Up @@ -124,10 +140,10 @@ func generateGlobalRegistryMirrorFile(mirror *mirrorConfig) ([]cabpkv1.File, err
}

func generateMirrorCACertFile(
config *mirrorConfig,
mirror *mirrorConfig,
globalMirror v1alpha1.GlobalImageRegistryMirror,
) []cabpkv1.File {
if config == nil || config.CACert == "" {
if mirror == nil || mirror.CACert == "" {
return nil
}
return []cabpkv1.File{
Expand All @@ -143,3 +159,42 @@ func generateMirrorCACertFile(
},
}
}

func generateContainerdRegistryConfigDropInFile() []cabpkv1.File {
return []cabpkv1.File{
{
Path: containerdRegistryConfigDropInFileOnRemote,
Content: string(containerdRegistryConfigDropIn),
Permissions: "0600",
},
}
}

func generateContainerdApplyPatchesScript() ([]cabpkv1.File, string, error) {
t, err := template.New("").Parse(string(containerdApplyConfigPatchesScript))
if err != nil {
return nil, "", fmt.Errorf("failed to parse go template: %w", err)
}

templateInput := struct {
TOMLMergeImage string
PatchDir string
}{
TOMLMergeImage: tomlMergeImage,
PatchDir: containerdPatchesDirOnRemote,
}

var b bytes.Buffer
err = t.Execute(&b, templateInput)
if err != nil {
return nil, "", fmt.Errorf("failed executing template: %w", err)
}

return []cabpkv1.File{
{
Path: containerdApplyPatchesScriptOnRemote,
Content: b.String(),
Permissions: "0700",
},
}, containerdApplyPatchesScriptOnRemoteCommand, nil
}
55 changes: 55 additions & 0 deletions pkg/handlers/generic/mutation/mirrors/mirror_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,58 @@ func Test_generateMirrorCACertFile(t *testing.T) {
})
}
}

func Test_generateContainerdRegistryConfigDropInFile(t *testing.T) {
want := []cabpkv1.File{
{
Path: "/etc/containerd/cre.d/registry-config.toml",
Owner: "",
Permissions: "0600",
Encoding: "",
Append: false,
Content: `[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
`,
},
}
file := generateContainerdRegistryConfigDropInFile()
assert.Equal(t, want, file)
}

func Test_generateContainerdApplyPatchesScript(t *testing.T) {
wantFile := []cabpkv1.File{
{
Path: "/etc/containerd/apply-patches.sh",
Owner: "",
Permissions: "0700",
Encoding: "",
Append: false,
//nolint:lll // just a long string
Content: `#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
declare -r TOML_MERGE_IMAGE="ghcr.io/mesosphere/toml-merge:v0.2.0"
if ! ctr --namespace k8s.io images check "name==${TOML_MERGE_IMAGE}" | grep "${TOML_MERGE_IMAGE}" >/dev/null; then
ctr --namespace k8s.io images pull "${TOML_MERGE_IMAGE}"
fi
cleanup() {
ctr images unmount "${tmp_ctr_mount_dir}" || true
}
trap 'cleanup' EXIT
readonly tmp_ctr_mount_dir="$(mktemp -d)"
ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}"
"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "/etc/containerd/cre.d/*.toml" /etc/containerd/config.toml
`,
},
}
wantCmd := "/bin/bash /etc/containerd/apply-patches.sh"
file, cmd, _ := generateContainerdApplyPatchesScript()
assert.Equal(t, wantFile, file)
assert.Equal(t, wantCmd, cmd)
}
20 changes: 20 additions & 0 deletions pkg/handlers/generic/mutation/mirrors/templates/containerd-apply-patches.sh.gotmpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'

declare -r TOML_MERGE_IMAGE="{{ .TOMLMergeImage }}"

if ! ctr --namespace k8s.io images check "name==${TOML_MERGE_IMAGE}" | grep "${TOML_MERGE_IMAGE}" >/dev/null; then
ctr --namespace k8s.io images pull "${TOML_MERGE_IMAGE}"
fi

cleanup() {
ctr images unmount "${tmp_ctr_mount_dir}" || true
}

trap 'cleanup' EXIT

readonly tmp_ctr_mount_dir="$(mktemp -d)"

ctr --namespace k8s.io images mount "${TOML_MERGE_IMAGE}" "${tmp_ctr_mount_dir}"
"${tmp_ctr_mount_dir}/usr/local/bin/toml-merge" -i --patch-file "{{ .PatchDir }}/*.toml" /etc/containerd/config.toml
2 changes: 2 additions & 0 deletions pkg/handlers/generic/mutation/mirrors/templates/containerd-registry-config-drop-in.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
52 changes: 52 additions & 0 deletions pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,19 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/certs.d/_default/hosts.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/cre.d/registry-config.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
ValueMatcher: gomega.ContainElement(
"/bin/bash /etc/containerd/apply-patches.sh",
),
},
},
Expand Down Expand Up @@ -101,6 +114,19 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/certs/mirror.pem",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/cre.d/registry-config.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
ValueMatcher: gomega.ContainElement(
"/bin/bash /etc/containerd/apply-patches.sh",
),
},
},
Expand Down Expand Up @@ -133,6 +159,19 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/certs.d/_default/hosts.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/cre.d/registry-config.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/preKubeadmCommands",
ValueMatcher: gomega.ContainElement(
"/bin/bash /etc/containerd/apply-patches.sh",
),
},
},
Expand Down Expand Up @@ -173,6 +212,19 @@ func TestGeneratePatches(
gomega.HaveKeyWithValue(
"path", "/etc/certs/mirror.pem",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/cre.d/registry-config.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/containerd/apply-patches.sh",
),
),
},
{
Operation: "add",
Path: "/spec/template/spec/preKubeadmCommands",
ValueMatcher: gomega.ContainElement(
"/bin/bash /etc/containerd/apply-patches.sh",
),
},
},
Expand Down

0 comments on commit c8ba1cf

Please sign in to comment.